r2914: rename .cl files
[clsql.git] / test-suite / acl-compat-tester.lisp
1 ;; tester.cl
2 ;; A test harness for Allegro CL.
3 ;;
4 ;; copyright (c) 1985-1986 Franz Inc, Alameda, CA
5 ;; copyright (c) 1986-2001 Franz Inc, Berkeley, CA - All rights reserved.
6 ;;
7 ;; This code is free software; you can redistribute it and/or
8 ;; modify it under the terms of the version 2.1 of
9 ;; the GNU Lesser General Public License as published by 
10 ;; the Free Software Foundation, as clarified by the Franz
11 ;; preamble to the LGPL found in
12 ;; http://opensource.franz.com/preamble.html.
13 ;;
14 ;; This code is distributed in the hope that it will be useful,
15 ;; but without any warranty; without even the implied warranty of
16 ;; merchantability or fitness for a particular purpose.  See the GNU
17 ;; Lesser General Public License for more details.
18 ;;
19 ;; Version 2.1 of the GNU Lesser General Public License can be
20 ;; found at http://opensource.franz.com/license.html.
21 ;; If it is not present, you can access it from
22 ;; http://www.gnu.org/copyleft/lesser.txt (until superseded by a newer
23 ;; version) or write to the Free Software Foundation, Inc., 59 Temple
24 ;; Place, Suite 330, Boston, MA  02111-1307  USA
25 ;;
26 ;;;; from the original ACL 6.1 sources:
27 ;; $Id: acl-compat-tester.lisp,v 1.1 2002/09/30 10:19:23 kevin Exp $
28
29
30 (defpackage :util.test
31   (:use :common-lisp)
32   (:shadow #:test)
33   (:export
34 ;;;; Control variables:
35    #:*break-on-test-failures*
36    #:*error-protect-tests*
37    #:*test-errors*
38    #:*test-successes*
39    #:*test-unexpected-failures*
40
41 ;;;; The test macros:
42    #:test
43    #:test-error
44    #:test-no-error
45    #:test-warning
46    #:test-no-warning
47    
48    #:with-tests
49    ))
50
51 (in-package :util.test)
52
53 #+cmu
54 (unless (find-class 'break nil)
55   (define-condition break (simple-condition) ()))
56
57 (define-condition simple-break (error simple-condition) ())
58
59 ;; the if* macro used in Allegro:
60 ;;
61 ;; This is in the public domain... please feel free to put this definition
62 ;; in your code or distribute it with your version of lisp.
63
64 (eval-when (:compile-toplevel :load-toplevel :execute)
65   (defvar if*-keyword-list '("then" "thenret" "else" "elseif")))
66
67 (defmacro if* (&rest args)
68    (do ((xx (reverse args) (cdr xx))
69         (state :init)
70         (elseseen nil)
71         (totalcol nil)
72         (lookat nil nil)
73         (col nil))
74        ((null xx)
75         (cond ((eq state :compl)
76                `(cond ,@totalcol))
77               (t (error "if*: illegal form ~s" args))))
78        (cond ((and (symbolp (car xx))
79                    (member (symbol-name (car xx))
80                            if*-keyword-list
81                            :test #'string-equal))
82               (setq lookat (symbol-name (car xx)))))
83
84        (cond ((eq state :init)
85               (cond (lookat (cond ((string-equal lookat "thenret")
86                                    (setq col nil
87                                          state :then))
88                                   (t (error
89                                       "if*: bad keyword ~a" lookat))))
90                     (t (setq state :col
91                              col nil)
92                        (push (car xx) col))))
93              ((eq state :col)
94               (cond (lookat
95                      (cond ((string-equal lookat "else")
96                             (cond (elseseen
97                                    (error
98                                     "if*: multiples elses")))
99                             (setq elseseen t)
100                             (setq state :init)
101                             (push `(t ,@col) totalcol))
102                            ((string-equal lookat "then")
103                             (setq state :then))
104                            (t (error "if*: bad keyword ~s"
105                                               lookat))))
106                     (t (push (car xx) col))))
107              ((eq state :then)
108               (cond (lookat
109                      (error
110                       "if*: keyword ~s at the wrong place " (car xx)))
111                     (t (setq state :compl)
112                        (push `(,(car xx) ,@col) totalcol))))
113              ((eq state :compl)
114               (cond ((not (string-equal lookat "elseif"))
115                      (error "if*: missing elseif clause ")))
116               (setq state :init)))))
117
118
119
120
121 (defvar *break-on-test-failures* nil
122   "When a test failure occurs, common-lisp:break is called, allowing
123 interactive debugging of the failure.")
124
125 (defvar *test-errors* 0
126   "The value is the number of test errors which have occurred.")
127 (defvar *test-successes* 0
128   "The value is the number of test successes which have occurred.")
129 (defvar *test-unexpected-failures* 0
130   "The value is the number of unexpected test failures which have occurred.")
131
132 (defvar *error-protect-tests* nil
133   "Protect each test from errors.  If an error occurs, then that will be
134 taken as a test failure unless test-error is being used.")
135
136 (defmacro test-values-errorset (form &optional announce catch-breaks)
137   ;; internal macro
138   (let ((g-announce (gensym))
139         (g-catch-breaks (gensym)))
140     `(let* ((,g-announce ,announce)
141             (,g-catch-breaks ,catch-breaks))
142        (handler-case (cons t (multiple-value-list ,form))
143          (condition (condition)
144            (if* (and (null ,g-catch-breaks)
145                      (typep condition 'simple-break))
146               then (break condition)
147             elseif ,g-announce
148               then (format *error-output* "~&Condition type: ~a~%"
149                            (class-of condition))
150                    (format *error-output* "~&Message: ~a~%" condition))
151            condition)))))
152
153 (defmacro test-values (form &optional announce catch-breaks)
154   ;; internal macro
155   (if* *error-protect-tests*
156      then `(test-values-errorset ,form ,announce ,catch-breaks)
157      else `(cons t (multiple-value-list ,form))))
158
159 (defmacro test (expected-value test-form
160                 &key (test #'eql test-given)
161                      (multiple-values nil multiple-values-given)
162                      (fail-info nil fail-info-given)
163                      (known-failure nil known-failure-given)
164
165 ;;;;;;;;;; internal, undocumented keywords:
166 ;;;; Note about these keywords: if they were documented, we'd have a
167 ;;;; problem, since they break the left-to-right order of evaluation.
168 ;;;; Specifically, errorset breaks it, and I don't see any way around
169 ;;;; that.  `errorset' is used by the old test.cl module (eg,
170 ;;;; test-equal-errorset).
171                      errorset
172                      reported-form
173                      (wanted-message nil wanted-message-given)
174                      (got-message nil got-message-given))
175   "Perform a single test.  `expected-value' is the reference value for the
176 test.  `test-form' is a form that will produce the value to be compared to
177 the expected-value.  If the values are not the same, then an error is
178 logged, otherwise a success is logged.
179
180 Normally the comparison of values is done with `eql'.  The `test' keyword
181 argument can be used to specify other comparison functions, such as eq,
182 equal,equalp, string=, string-equal, etc.
183
184 Normally, only the first return value from the test-form is considered,
185 however if `multiple-values' is t, then all values returned from test-form
186 are considered.
187
188 `fail-info' allows more information to be printed with a test failure.
189
190 `known-failure' marks the test as a known failure.  This allows for
191 programs that do regression analysis on the output from a test run to
192 discriminate on new versus known failures."
193   `(test-check
194     :expected-result ,expected-value
195     :test-results
196     (,(if errorset 'test-values-errorset 'test-values) ,test-form t)
197     ,@(when test-given `(:predicate ,test))
198     ,@(when multiple-values-given `(:multiple-values ,multiple-values))
199     ,@(when fail-info-given `(:fail-info ,fail-info))
200     ,@(when known-failure-given `(:known-failure ,known-failure))
201     :test-form ',(if reported-form reported-form test-form)
202     ,@(when wanted-message-given `(:wanted-message ,wanted-message))
203     ,@(when got-message-given `(:got-message ,got-message))))
204
205 (defmethod conditionp ((thing condition)) t)
206 (defmethod conditionp ((thing t)) nil)
207
208 (defmacro test-error (form &key announce
209                                 catch-breaks
210                                 (fail-info nil fail-info-given)
211                                 (known-failure nil known-failure-given)
212                                 (condition-type ''simple-error)
213                                 (include-subtypes nil include-subtypes-given)
214                                 (format-control nil format-control-given)
215                                 (format-arguments nil format-arguments-given))
216   "Test that `form' signals an error. The order of evaluation of the
217 arguments is keywords first, then test form.
218
219 If `announce' is non-nil, then cause the error message to be printed.
220
221 The `catch-breaks' is non-nil then consider a call to common-lisp:break an
222 `error'.
223
224 `fail-info' allows more information to be printed with a test failure.
225
226 `known-failure' marks the test as a known failure.  This allows for
227 programs that do regression analysis on the output from a test run to
228 discriminate on new versus known failures.
229
230 If `condition-type' is non-nil, it should be a symbol naming a condition
231 type, which is used to check against the signalled condition type.  The
232 test will fail if they do not match.
233
234 `include-subtypes', used with `condition-type', can be used to match a
235 condition to an entire subclass of the condition type hierarchy.
236
237 `format-control' and `format-arguments' can be used to check the error
238 message itself."
239   (let ((g-announce (gensym))
240         (g-catch-breaks (gensym))
241         (g-fail-info (gensym))
242         (g-known-failure (gensym))
243         (g-condition-type (gensym))
244         (g-include-subtypes (gensym))
245         (g-format-control (gensym))
246         (g-format-arguments (gensym))
247         (g-c (gensym)))
248     `(let* ((,g-announce ,announce)
249             (,g-catch-breaks ,catch-breaks)
250             ,@(when fail-info-given `((,g-fail-info ,fail-info)))
251             ,@(when known-failure-given `((,g-known-failure ,known-failure)))
252             (,g-condition-type ,condition-type)
253             ,@(when include-subtypes-given
254                 `((,g-include-subtypes ,include-subtypes)))
255             ,@(when format-control-given
256                 `((,g-format-control ,format-control)))
257             ,@(when format-arguments-given
258                 `((,g-format-arguments ,format-arguments)))
259             (,g-c (test-values-errorset ,form ,g-announce ,g-catch-breaks)))
260        (test-check
261         :predicate #'eq
262         :expected-result t
263         :test-results
264         (test-values (and (conditionp ,g-c)
265                           ,@(if* include-subtypes-given
266                                then `((if* ,g-include-subtypes
267                                          then (typep ,g-c ,g-condition-type)
268                                          else (eq (class-of ,g-c)
269                                                   (find-class
270                                                    ,g-condition-type))))
271                                else `((eq (class-of ,g-c)
272                                           (find-class ,g-condition-type))))
273                           ,@(when format-control-given
274                               `((or
275                                  (null ,g-format-control)
276                                  (string=
277                                   (concatenate 'simple-string
278                                     "~1@<" ,g-format-control "~:@>")
279                                   (simple-condition-format-control ,g-c)))))
280                           ,@(when format-arguments-given
281                               `((or
282                                  (null ,g-format-arguments)
283                                  (equal
284                                   ,g-format-arguments
285                                   (simple-condition-format-arguments ,g-c))))))
286                      t)
287         :test-form ',form
288         ,@(when fail-info-given `(:fail-info ,g-fail-info))
289         ,@(when known-failure-given `(:known-failure ,g-known-failure))
290         :condition-type ,g-condition-type
291         :condition ,g-c
292         ,@(when include-subtypes-given
293             `(:include-subtypes ,g-include-subtypes))
294         ,@(when format-control-given
295             `(:format-control ,g-format-control))
296         ,@(when format-arguments-given
297             `(:format-arguments ,g-format-arguments))))))
298
299 (defmacro test-no-error (form &key announce
300                                    catch-breaks
301                                    (fail-info nil fail-info-given)
302                                    (known-failure nil known-failure-given))
303   "Test that `form' does not signal an error.  The order of evaluation of
304 the arguments is keywords first, then test form.
305
306 If `announce' is non-nil, then cause the error message to be printed.
307
308 The `catch-breaks' is non-nil then consider a call to common-lisp:break an
309 `error'.
310
311 `fail-info' allows more information to be printed with a test failure.
312
313 `known-failure' marks the test as a known failure.  This allows for
314 programs that do regression analysis on the output from a test run to
315 discriminate on new versus known failures."
316   (let ((g-announce (gensym))
317         (g-catch-breaks (gensym))
318         (g-fail-info (gensym))
319         (g-known-failure (gensym))
320         (g-c (gensym)))
321     `(let* ((,g-announce ,announce)
322             (,g-catch-breaks ,catch-breaks)
323             ,@(when fail-info-given `((,g-fail-info ,fail-info)))
324             ,@(when known-failure-given `((,g-known-failure ,known-failure)))
325             (,g-c (test-values-errorset ,form ,g-announce ,g-catch-breaks)))
326        (test-check
327         :predicate #'eq
328         :expected-result t
329         :test-results (test-values (not (conditionp ,g-c)))
330         :test-form ',form
331         :condition ,g-c
332         ,@(when fail-info-given `(:fail-info ,g-fail-info))
333         ,@(when known-failure-given `(:known-failure ,g-known-failure))))))
334
335 (defvar *warn-cookie* (cons nil nil))
336
337 (defmacro test-warning (form &key fail-info known-failure)
338   "Test that `form' signals a warning.  The order of evaluation of
339 the arguments is keywords first, then test form.
340
341 `fail-info' allows more information to be printed with a test failure.
342
343 `known-failure' marks the test as a known failure.  This allows for
344 programs that do regression analysis on the output from a test run to
345 discriminate on new versus known failures."
346   (let ((g-fail-info (gensym))
347         (g-known-failure (gensym))
348         (g-value (gensym)))
349     `(let* ((,g-fail-info ,fail-info)
350             (,g-known-failure ,known-failure)
351             (,g-value (test-values-errorset ,form nil t)))
352        (test
353         *warn-cookie*
354         (if* (or (typep ,g-value 'simple-warning) (typep ,g-value 'warning))
355            then *warn-cookie*
356            else ;; test produced no warning
357                 nil)
358         :test #'eq
359         :reported-form ,form ;; quoted by test macro
360         :wanted-message "a warning"
361         :got-message "no warning"
362         :fail-info ,g-fail-info
363         :known-failure ,g-known-failure))))
364
365 (defmacro test-no-warning (form &key fail-info known-failure)
366   "Test that `form' does not signal a warning.  The order of evaluation of
367 the arguments is keywords first, then test form.
368
369 `fail-info' allows more information to be printed with a test failure.
370
371 `known-failure' marks the test as a known failure.  This allows for
372 programs that do regression analysis on the output from a test run to
373 discriminate on new versus known failures."
374   (let ((g-fail-info (gensym))
375         (g-known-failure (gensym))
376         (g-value (gensym)))
377     `(let* ((,g-fail-info ,fail-info)
378             (,g-known-failure ,known-failure)
379             (,g-value (test-values-errorset ,form nil t)))
380        (test
381         *warn-cookie*
382         (if* (or (typep ,g-value 'simple-warning) (typep ,g-value 'warning))
383            then nil ;; test produced warning
384            else *warn-cookie*)
385         :test #'eq
386         :reported-form ',form
387         :wanted-message "no warning"
388         :got-message "a warning"
389         :fail-info ,g-fail-info
390         :known-failure ,g-known-failure))))
391
392 (defvar *announce-test* nil) ;; if true announce each test that was done
393
394 (defmacro errorset (form &optional announce catch-breaks)
395   ;; Evaluate FORM, and if there are no errors and FORM returns
396   ;; values v1,v2,...,vn, then return values t,v1,v2,...,vn.  If an
397   ;; error occurs while evaluating FORM, then return nil immediately.
398   ;; If ANNOUNCE is t, then the error message will be printed out.
399   (if catch-breaks
400       `(handler-case (values-list (cons t (multiple-value-list ,form)))
401          (error (condition)
402            (declare (ignore-if-unused condition))
403            ,@(if announce `((format *error-output* "~&Error: ~a~%" condition)))
404            nil)
405          (simple-break (condition)
406            (declare (ignore-if-unused condition))
407            ,@(if announce `((format *error-output* "~&Warning: ~a~%" condition))
408 )
409            nil))
410     `(handler-case (values-list (cons t (multiple-value-list ,form)))
411        (error (condition)
412          (declare (ignore-if-unused condition))
413          ,@(if announce `((format *error-output* "~&Error: ~a~%" condition)))
414          nil))))
415
416 (defun test-check (&key (predicate #'eql)
417                         expected-result test-results test-form
418                         multiple-values fail-info known-failure
419                         wanted-message got-message condition-type condition
420                         include-subtypes format-control format-arguments
421                    &aux fail predicate-failed got wanted)
422   ;; for debugging large/complex test sets:
423   (when *announce-test*
424     (format t "Just did test ~s~%" test-form)
425     (force-output))
426   
427   ;; this is an internal function
428   (flet ((check (expected-result result)
429            (let* ((results
430                    (multiple-value-list
431                     (errorset (funcall predicate expected-result result) t)))
432                   (failed (null (car results))))
433              (if* failed
434                 then (setq predicate-failed t)
435                      nil
436                 else (cadr results)))))
437     (when (conditionp test-results)
438       (setq condition test-results)
439       (setq test-results nil))
440     (when (null (car test-results))
441       (setq fail t))
442     (if* (and (not fail) (not multiple-values))
443        then ;; should be a single result
444             ;; expected-result is the single result wanted
445             (when (not (and (cdr test-results)
446                             (check expected-result (cadr test-results))))
447               (setq fail t))
448             (when (and (not fail) (cddr test-results))
449               (setq fail 'single-got-multiple))
450        else ;; multiple results wanted
451             ;; expected-result is a list of results, each of which
452             ;; should be checked against the corresponding test-results
453             ;; using the predicate
454             (do ((got (cdr test-results) (cdr got))
455                  (want expected-result (cdr want)))
456                 ((or (null got) (null want))
457                  (when (not (and (null want) (null got)))
458                    (setq fail t)))
459               (when (not (check (car got) (car want)))
460                 (return (setq fail t)))))
461     (if* fail
462        then (when (not known-failure)
463               (format *error-output*
464                       "~& * * * UNEXPECTED TEST FAILURE * * *~%")
465               (incf *test-unexpected-failures*))
466             (format *error-output* "~&Test failed: ~@[known failure: ~*~]~s~%"
467                     known-failure test-form)
468             (if* (eq 'single-got-multiple fail)
469                then (format
470                      *error-output*
471                      "~
472 Reason: additional value were returned from test form.~%")
473              elseif predicate-failed
474                then (format *error-output* "Reason: predicate error.~%")
475              elseif (null (car test-results))
476                then (format *error-output* "~
477 Reason: an error~@[ (of type `~s')~] was detected.~%"
478                             (when condition (class-of condition)))
479              elseif condition
480                then (if* (not (conditionp condition))
481                        then (format *error-output* "~
482 Reason: expected but did not detect an error of type `~s'.~%"
483                                     condition-type)
484                      elseif (null condition-type)
485                        then (format *error-output* "~
486 Reason: detected an unexpected error of type `~s':
487         ~a.~%"
488                                     (class-of condition)
489                                     condition)
490                      elseif (not (if* include-subtypes
491                                     then (typep condition condition-type)
492                                     else (eq (class-of condition)
493                                              (find-class condition-type))))
494                        then (format *error-output* "~
495 Reason: detected an incorrect condition type.~%")
496                             (format *error-output*
497                                     "  wanted: ~s~%" condition-type)
498                             (format *error-output*
499                                     "     got: ~s~%" (class-of condition))
500                      elseif (and format-control
501                                  (not (string=
502                                        (setq got
503                                          (concatenate 'simple-string
504                                            "~1@<" format-control "~:@>"))
505                                        (setq wanted
506                                          (simple-condition-format-control
507                                           condition)))))
508                        then ;; format control doesn't match
509                             (format *error-output* "~
510 Reason: the format-control was incorrect.~%")
511                             (format *error-output* "  wanted: ~s~%" wanted)
512                             (format *error-output* "     got: ~s~%" got)
513                      elseif (and format-arguments
514                                  (not (equal
515                                        (setq got format-arguments)
516                                        (setq wanted
517                                          (simple-condition-format-arguments
518                                           condition)))))
519                        then (format *error-output* "~
520 Reason: the format-arguments were incorrect.~%")
521                             (format *error-output* "  wanted: ~s~%" wanted)
522                             (format *error-output* "     got: ~s~%" got)
523                        else ;; what else????
524                             (error "internal-error"))
525                else (let ((*print-length* 50)
526                           (*print-level* 10))
527                       (if* wanted-message
528                          then (format *error-output*
529                                       "  wanted: ~a~%" wanted-message)
530                          else (if* (not multiple-values)
531                                  then (format *error-output*
532                                               "  wanted: ~s~%"
533                                               expected-result)
534                                  else (format
535                                        *error-output*
536                                        "  wanted values: ~{~s~^, ~}~%"
537                                        expected-result)))
538                       (if* got-message
539                          then (format *error-output*
540                                       "     got: ~a~%" got-message)
541                          else (if* (not multiple-values)
542                                  then (format *error-output* "     got: ~s~%"
543                                        (second test-results))
544                                  else (format
545                                        *error-output*
546                                        "     got values: ~{~s~^, ~}~%"
547                                        (cdr test-results))))))
548             (when fail-info
549               (format *error-output* "Additional info: ~a~%" fail-info))
550             (incf *test-errors*)
551             (when *break-on-test-failures*
552               (break "~a is non-nil." '*break-on-test-failures*))
553        else (when known-failure
554               (format *error-output*
555                       "~&Expected test failure for ~s did not occur.~%"
556                       test-form)
557               (when fail-info
558                 (format *error-output* "Additional info: ~a~%" fail-info))
559               (setq fail t))
560             (incf *test-successes*))
561     (not fail)))
562
563 (defmacro with-tests ((&key (name "unnamed")) &body body)
564   (let ((g-name (gensym)))
565     `(flet ((doit () ,@body))
566        (let ((,g-name ,name)
567              (*test-errors* 0)
568              (*test-successes* 0)
569              (*test-unexpected-failures* 0))
570          (format *error-output* "Begin ~a test~%" ,g-name)
571          (if* *break-on-test-failures*
572             then (doit)
573             else (handler-case (doit)
574                    (error (c)
575                      (format
576                       *error-output*
577                       "~
578 ~&Test ~a aborted by signalling an uncaught error:~%~a~%"
579                       ,g-name c))))
580          #+allegro
581          (let ((state (sys:gsgc-switch :print)))
582            (setf (sys:gsgc-switch :print) nil)
583            (format t "~&**********************************~%" ,g-name)
584            (format t "End ~a test~%" ,g-name)
585            (format t "Errors detected in this test: ~s " *test-errors*)
586            (unless (zerop *test-unexpected-failures*)
587              (format t "UNEXPECTED: ~s" *test-unexpected-failures*))
588            (format t "~%Successes this test:~s~%" *test-successes*)
589            (setf (sys:gsgc-switch :print) state))
590          #-allegro
591          (progn
592            (format t "~&**********************************~%" ,g-name)
593            (format t "End ~a test~%" ,g-name)
594            (format t "Errors detected in this test: ~s " *test-errors*)
595            (unless (zerop *test-unexpected-failures*)
596              (format t "UNEXPECTED: ~s" *test-unexpected-failures*))
597            (format t "~%Successes this test:~s~%" *test-successes*))
598          ))))
599
600 (provide :tester #+module-versions 1.1)