1 ;;;; -*- Mode: LISP; Syntax: ANSI-Common-Lisp; Base: 10 -*-
2 ;;;; ======================================================================
3 ;;;; File: test-init.lisp
4 ;;;; Authors: Marcus Pearce <m.t.pearce@city.ac.uk>, Kevin Rosenberg
5 ;;;; Created: 30/03/2004
7 ;;;; Initialisation utilities for running regression tests on CLSQL.
9 ;;;; This file is part of CLSQL.
11 ;;;; CLSQL users are granted the rights to distribute and use this software
12 ;;;; as governed by the terms of the Lisp Lesser GNU Public License
13 ;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL.
14 ;;;; ======================================================================
16 (in-package #:clsql-tests)
18 (defvar *report-stream* *standard-output* "Stream to send text report.")
19 (defvar *sexp-report-stream* nil "Stream to send sexp report.")
20 (defvar *rt-internal*)
22 (defvar *rt-connection*)
30 ;; Below must be set as nil since test-i18n.lisp is not loaded on all platforms.
31 (defvar *rt-i18n* nil)
33 (defvar *test-database-type* nil)
34 (defvar *test-database-underlying-type* nil)
35 (defvar *test-database-user* nil)
36 (defvar *test-false-database-user* "adsfjalsdkfjlakjsdfl"
37 "For testing ownership, a user that isn't the owner.")
38 (defvar *test-start-utime* nil)
39 (defvar *test-connection-spec* nil)
40 (defvar *test-connection-db-type* nil)
41 (defvar *test-report-width* 80 "Width of test report in ems.")
44 (defun find-test-connection-spec (db-type &key position)
46 (db-type-spec db-type (read-specs))))
50 (db-type *test-database-type* db-type-p)
54 (or (and (null db-type-p) *test-connection-spec*)
55 (find-test-connection-spec db-type :position position))))
56 (when *default-database*
57 (disconnect :database *default-database*))
58 (setf *test-database-type* db-type
61 ((member db-type '(:oracle :odbc :aodbc)) (second spec))
62 ((>= (length spec) 3) (third spec)))
63 *test-connection-spec* spec
67 :database-type db-type
71 *test-database-underlying-type*
72 (clsql-sys:database-underlying-type *default-database*))
75 (defun test-setup-database (db-type &key (spec (find-test-connection-spec db-type)))
76 (when (clsql-sys:db-backend-has-create/destroy-db? db-type)
77 (ignore-errors (destroy-database spec :database-type db-type))
78 (ignore-errors (create-database spec :database-type db-type)))
80 ;; Connect to the database
81 (test-connect :db-type db-type :spec spec)
82 ;; Ensure database is empty
83 (truncate-database :database *default-database*)
85 ;; If Postgres, turn off notices to console
86 (when (eql db-type :postgresql)
87 (clsql:execute-command "SET client_min_messages = WARNING"))
91 (defun default-suites ()
92 "The default list of tests to run."
93 (append *rt-connection* *rt-basic* *rt-fddl* *rt-fdml*
94 *rt-ooddl* *rt-oodml* *rt-syntax* *rt-time* *rt-i18n*))
96 (defun internal-suites ()
97 "The default internal suites that should run without any specific backend"
98 (append *rt-internal* *rt-pool*))
101 (defvar *error-count* 0)
102 (defvar *error-list* nil)
104 (defun run-function-append-report-file (function report-file)
105 (let* ((report-path (etypecase report-file
106 (pathname report-file)
107 (string (parse-namestring report-file))))
108 (sexp-report-path (make-pathname :defaults report-path
110 (with-open-file (rs report-path :direction :output
112 :if-does-not-exist :create)
113 (with-open-file (srs sexp-report-path :direction :output
115 :if-does-not-exist :create)
116 (funcall function :report-stream rs :sexp-report-stream srs)))))
118 (defun run-tests-append-report-file (report-file)
119 (run-function-append-report-file 'run-tests report-file))
122 (defun run-tests (&key (report-stream *standard-output*) (sexp-report-stream nil)
123 (suites (append (internal-suites) (default-suites))))
124 ;; clear SQL-OUTPUT cache
125 (setq clsql-sys::*output-hash* (make-hash-table :test #'equal))
126 (setf *test-database-underlying-type* nil)
127 (let ((specs (read-specs))
128 (*report-stream* report-stream)
129 (*sexp-report-stream* sexp-report-stream)
133 (warn "Not running tests because test configuration file is missing")
134 (return-from run-tests :skipped))
135 (load-necessary-systems specs)
136 ;;run the internal suites
137 (do-tests-for-internals :suites (intersection suites (internal-suites)))
138 ;; run backend-specific tests
139 (let ((suites (intersection suites (default-suites))))
141 (dolist (db-type +all-db-types+)
142 (dolist (spec (db-type-spec db-type specs))
143 (format report-stream "~%~%Start Running Tests Against: ~A ~A~%~%" db-type (ignore-errors (subseq spec 0 2)))
144 (do-tests-for-backend db-type spec :suites suites)
145 (format report-stream "~%~%Finished Running Tests Against: ~A ~A~%~%" db-type (ignore-errors (subseq spec 0 2)))))))
146 (zerop *error-count*)))
148 (defun load-necessary-systems (specs)
149 (dolist (db-type +all-db-types+)
150 (when (db-type-spec db-type specs)
151 (clsql-sys:initialize-database-type :database-type db-type))))
153 (defun write-report-banner (report-type db-type stream db-name)
156 ******************************************************************************
157 *** CLSQL ~A begun at ~A
160 *** Database ~:@(~A~)
161 *** Type: ~:@(~A~) backend~A.
162 ******************************************************************************
167 (clsql:utime->time (get-universal-time)))
168 (lisp-implementation-type)
169 (lisp-implementation-version)
173 (if (not (eq db-type *test-database-underlying-type*))
174 (format nil " with underlying type ~:@(~A~)"
175 *test-database-underlying-type*)
179 (defun do-tests-for-internals (&key (suites (internal-suites)))
180 (write-report-banner "Test Suite" "CLSQL Internals" *report-stream*
182 (%do-tests suites nil))
184 (defun %do-tests (test-forms db-type)
185 (regression-test:rem-all-tests)
186 (dolist (test-form test-forms)
189 (let* ((cl:*print-right-margin* *test-report-width*)
190 (remaining (regression-test:do-tests *report-stream*)))
191 (when (regression-test:pending-tests)
192 (incf *error-count* (length remaining))))
194 (let ((sexp-error (list db-type
195 *test-database-underlying-type*
198 (regression-test:pending-tests)
199 (lisp-implementation-type)
200 (lisp-implementation-version)
202 (when *sexp-report-stream*
203 (write sexp-error :stream *sexp-report-stream* :readably t))
204 (push sexp-error *error-list*))
207 (defun do-tests-for-backend (db-type spec &key
208 (suites (default-suites)) )
209 (test-setup-database db-type :spec spec)
211 (multiple-value-bind (test-forms skip-tests)
212 (compute-tests-for-backend db-type *test-database-underlying-type* :suites suites)
214 (write-report-banner "Test Suite" db-type *report-stream*
215 (database-name-from-spec spec db-type))
217 (%do-tests test-forms db-type)
219 (format *report-stream* "~&~D of ~D Tests skipped:"
223 (let ((max-test-name (length (symbol-name (caar skip-tests)))))
224 (dolist (skipped (cdr skip-tests))
225 (let ((len (length (symbol-name (car skipped)))))
226 (when (> len max-test-name)
227 (setq max-test-name len))))
228 (let ((fmt (format nil "~~& ~~~DA ~~A~~%" max-test-name)))
229 (dolist (skipped skip-tests)
230 ;; word-wrap the reason string field
231 (let* ((test (car skipped))
232 (reason (cdr skipped))
233 ;; (rlen (length reason))
234 (rwidth (max 20 (- (or *test-report-width* 80) max-test-name 3)))
235 (rwords (clsql-sys::delimited-string-to-list reason #\space t))
236 (rformat (format nil "~~{~~<~%~~1,~D:;~~A~~> ~~}" rwidth))
237 (rwrapped (format nil rformat rwords))
238 (rlines (clsql-sys::delimited-string-to-list rwrapped #\Newline t)))
239 (dolist (rline rlines)
240 (format *report-stream* fmt (if test
246 (format *report-stream* " None~%")))
250 (defun compute-tests-for-backend (db-type db-underlying-type
251 &key (suites (default-suites)))
252 (let ((test-forms '())
254 (dolist (test-form (if (listp suites) suites (list suites)))
255 (let ((test (second test-form)))
257 ((and (not (eql db-underlying-type :mysql))
258 (clsql-sys:in test :connection/query-command
259 :basic/reallybigintegers/1
260 :connection/pool/procedure-mysql))
261 (push (cons test "known to work only in MySQL as yet.") skip-tests))
262 ((and (null (clsql-sys:db-type-has-views? db-underlying-type))
263 (clsql-sys:in test :fddl/view/1 :fddl/view/2 :fddl/view/3 :fddl/view/4))
264 (push (cons test "views not supported.") skip-tests))
265 ((and (null (clsql-sys:db-type-has-boolean-where? db-underlying-type))
266 (clsql-sys:in test :fdml/select/11 :oodml/select/5))
267 (push (cons test "boolean where not supported.") skip-tests))
268 ((and (null (clsql-sys:db-type-has-subqueries? db-underlying-type))
269 (clsql-sys:in test :fdml/select/5 :fdml/select/10
270 :fdml/select/32 :fdml/select/33))
271 (push (cons test "subqueries not supported.") skip-tests))
272 ((and (null (clsql-sys:db-type-transaction-capable? db-underlying-type
274 (clsql-sys:in test :fdml/transaction/1 :fdml/transaction/2 :fdml/transaction/3 :fdml/transaction/4))
275 (push (cons test "transactions not supported.") skip-tests))
276 ((and (null (clsql-sys:db-type-has-fancy-math? db-underlying-type))
277 (clsql-sys:in test :fdml/select/1))
278 (push (cons test "fancy math not supported.") skip-tests))
279 ((and (eql *test-database-type* :sqlite)
280 (clsql-sys:in test :fddl/view/4 :fdml/select/10
281 :fdml/select/21 :fdml/select/32
283 (push (cons test "not supported by sqlite.") skip-tests))
284 ((and (eql *test-database-type* :sqlite3)
285 (clsql-sys:in test :fddl/view/4 :fdml/select/10
286 :fdml/select/21 :fdml/select/32
288 (push (cons test "not supported by sqlite3.") skip-tests))
289 ((and (not (clsql-sys:db-type-has-bigint? db-type))
290 (clsql-sys:in test :basic/bigint/1))
291 (push (cons test "bigint not supported.") skip-tests))
292 ((and (eql *test-database-underlying-type* :mysql)
293 (clsql-sys:in test :fdml/select/26))
294 (push (cons test "string table aliases not supported on all MySQL versions.") skip-tests))
295 ((and (eql *test-database-underlying-type* :mysql)
296 (clsql-sys:in test :fdml/select/22 :fdml/query/5
297 :fdml/query/7 :fdml/query/8))
298 (push (cons test "not supported by mysql.") skip-tests))
299 ((and (null (clsql-sys:db-type-has-union? db-underlying-type))
300 (clsql-sys:in test :fdml/query/6 :fdml/select/31))
301 (push (cons test "union not supported") skip-tests))
302 ((and (eq *test-database-type* :oracle)
303 (clsql-sys:in test :fdml/query/8 :fdml/select/21
305 (push (cons test "syntax not supported.") skip-tests))
306 ((and (eq *test-database-type* :odbc)
307 (eq *test-database-underlying-type* :postgresql)
308 (clsql-sys:in test :fddl/owner/1 :fddl/owner/table
309 :fddl/owner/attributes
310 :fddl/owner/attribute-types
312 :fddl/owner/sequence))
313 (push (cons test "table ownership not supported by postgresql odbc driver.") skip-tests))
314 ((and (not (member *test-database-underlying-type*
315 '(:postgresql :oracle)))
316 (clsql-sys:in test :fddl/owner/1 :fddl/owner/table
317 :fddl/owner/attributes
318 :fddl/owner/attribute-types
320 :fddl/owner/sequence))
321 (push (cons test "table ownership not supported.") skip-tests))
322 ((and (null (clsql-sys:db-type-has-intersect? db-underlying-type))
323 (clsql-sys:in test :fdml/query/7))
324 (push (cons test "intersect not supported.") skip-tests))
325 ((and (null (clsql-sys:db-type-has-except? db-underlying-type))
326 (clsql-sys:in test :fdml/query/8))
327 (push (cons test "except not supported.") skip-tests))
328 ((and (eq *test-database-underlying-type* :mssql)
329 (clsql-sys:in test :fdml/select/9))
330 (push (cons test "mssql uses integer math for AVG.") skip-tests))
331 ((and (not (member *test-database-underlying-type*
332 '(:postgresql :mysql :sqlite3 )))
333 (clsql-sys:in test :fdml/select/37 :fdml/select/38))
334 (push (cons test "LIMIT keyword not supported in SELECT.") skip-tests))
335 ((and (not (clsql-sys:db-type-has-auto-increment? db-underlying-type))
336 (clsql-sys:in test :oodml/select/12 :oodml/select/13 :oodml/select/14
337 :oodml/select/15 :oodml/select/16 :oodml/select/17
338 :oodml/select/18 :oodml/select/19 :oodml/select/20
339 :oodml/select/21 :oodml/select/22 :oodml/select/23
340 :oodml/update-records/4 :oodml/update-records/4-slots
341 :oodml/update-records/5 :oodml/update-records/5-slots
342 :oodml/update-records/6 :oodml/update-records/7
343 :oodml/update-records/8 :oodml/update-records/9
344 :oodml/update-records/9-slots :oodml/update-records/10
345 :oodml/update-records/11 :OODML/UPDATE-RECORDS/12 :oodml/update-instance/3
346 :oodml/update-instance/4 :oodml/update-instance/5
347 :oodml/update-instance/6 :oodml/update-instance/7
348 :oodml/db-auto-sync/3 :oodml/db-auto-sync/4))
349 (push (cons test ":auto-increment not supported.") skip-tests))
350 ((and (not (member *test-database-underlying-type*
351 '(:postgresql :postgresql-socket)))
353 :time/pg/fdml/usec :time/pg/oodml/no-usec :time/pg/oodml/usec))
354 (push (cons test "Postgres specific test.")
356 ((and (eql *test-database-type* :postgresql-socket3)
357 (clsql-sys:in test :BASIC/SELECT/2 :basic/select/3))
358 (push (cons test "Postgres-socket3 always auto types")
360 ((and (eql *test-database-type* :postgresql-socket3)
361 (clsql-sys:in test :fdml/select/18))
362 (push (cons test "Postgres-socket3 doesnt support attribute based type coersion")
364 ((and (eql *test-database-type* :postgresql-socket3)
365 (clsql-sys:in test :basic/map/1 :basic/map/2 :basic/map/3 :basic/map/4
366 :basic/do/1 :basic/do/2 :fdml/do-query/1 :fdml/map-query/1
367 :fdml/map-query/2 :fdml/map-query/3 :fdml/map-query/4 :fdml/loop/1
368 :fdml/loop/2 :fdml/loop/3
370 (push (cons test "postgresql-socket3 doesnt support cursoring interface")
372 ((and (member *test-database-underlying-type* '(:mysql))
373 (clsql-sys:in test :time/cross-platform/msec
374 :time/cross-platform/usec/no-tz :time/cross-platform/usec/tz))
375 (push (cons test "MySQL doesn't support fractional seconds on timestamp columns (http://forge.mysql.com/worklog/task.php?id=946).")
377 ((and (member *test-database-underlying-type* '(:mssql))
378 (clsql-sys:in test :time/cross-platform/usec/no-tz :time/cross-platform/usec/tz))
379 (push (cons test "MSSQL doesn't support micro-seconds on datetime columns.")
382 (push test-form test-forms)))))
383 (values (nreverse test-forms) (nreverse skip-tests))))
385 (defun rapid-load (type &optional (position 0))
386 "Rapid load for interactive testing."
389 :spec (find-test-connection-spec type :position position))
393 (rapid-load :postgresql))
399 (rapid-load :oracle))