From 3199369942d2e5ab4f5b060c2c6b655caf505944 Mon Sep 17 00:00:00 2001 From: "Kevin M. Rosenberg" Date: Wed, 7 May 2003 21:57:10 +0000 Subject: [PATCH] r4870: *** empty log message *** --- Makefile | 32 +++ class-support.lisp | 10 +- sql-create.lisp => create-sql.lisp | 73 ++++--- data-structures.lisp | 4 +- parse-2002.lisp | 179 ++++------------- parse-common.lisp | 299 ++++++++++++++++++----------- parse-macros.lisp | 10 +- run-tests.lisp | 27 +++ umlisp.asd | 4 +- 9 files changed, 333 insertions(+), 305 deletions(-) create mode 100644 Makefile rename sql-create.lisp => create-sql.lisp (89%) create mode 100644 run-tests.lisp diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..dfa7a11 --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +.PHONY: all clean test test-acl test-sbcl + +test-file:=`pwd`/run-tests.lisp +all: + +clean: + @find . -type f -name "*.fasl*" -or -name "*.ufsl" -or -name "*.x86f" \ + -or -name "*.fas" -or -name "*.pfsl" -or -name "*.dfsl" \ + -or -name "*~" -or -name ".#*" -or -name "#*#" | xargs rm -f + +test: test-alisp + +test-alisp: + alisp8 -q -L $(test-file) + +test-mlisp: + mlisp -q -L $(test-file) + +test-sbcl: + sbcl --noinform --disable-debugger --userinit $(test-file) + +test-cmucl: + lisp -init $(test-file) + +test-lw: + lw-console -init $(test-file) + +test-scl: + scl -init $(test-file) + +test-clisp: + clisp -norc -q -i $(test-file) diff --git a/class-support.lisp b/class-support.lisp index 4b88f84..f78ed7e 100644 --- a/class-support.lisp +++ b/class-support.lisp @@ -2,12 +2,12 @@ ;;;; ************************************************************************* ;;;; FILE IDENTIFICATION ;;;; -;;;; Name: classes-support.lisp -;;;; Purpose: Class defintions for UMLisp -;;;; Programmer: Kevin M. Rosenberg -;;;; Date Started: Apr 2000 +;;;; Name: classes-support.lisp +;;;; Purpose: Support for UMLisp classes +;;;; Author: Kevin M. Rosenberg +;;;; Date Started: Apr 2000 ;;;; -;;;; $Id: class-support.lisp,v 1.3 2003/05/06 02:15:41 kevin Exp $ +;;;; $Id: class-support.lisp,v 1.4 2003/05/07 21:57:06 kevin Exp $ ;;;; ;;;; This file, part of UMLisp, is ;;;; Copyright (c) 2000-2002 by Kevin M. Rosenberg, M.D. diff --git a/sql-create.lisp b/create-sql.lisp similarity index 89% rename from sql-create.lisp rename to create-sql.lisp index 49f550e..e81440d 100644 --- a/sql-create.lisp +++ b/create-sql.lisp @@ -7,7 +7,7 @@ ;;;; Author: Kevin M. Rosenberg ;;;; Date Started: Apr 2000 ;;;; -;;;; $Id: sql-create.lisp,v 1.21 2003/05/06 08:15:47 kevin Exp $ +;;;; $Id: create-sql.lisp,v 1.1 2003/05/07 21:57:06 kevin Exp $ ;;;; ;;;; This file, part of UMLisp, is ;;;; Copyright (c) 2000-2002 by Kevin M. Rosenberg, M.D. @@ -16,7 +16,7 @@ ;;;; as governed by the terms of the GNU General Public License. ;;;; ************************************************************************* -(in-package :umlisp) +(in-package #:umlisp) (eval-when (:compile-toplevel) (declaim (optimize (speed 3) (safety 1) (compilation-speed 0) (debug 3)))) @@ -34,7 +34,7 @@ (format nil "~a (~a)" sqltype (cmax c)) sqltype)))))) (format nil "CREATE TABLE ~a (~{~a~^,~})" (table file) - (mapcar col-func (ucols-for-ufile file))))) + (mapcar col-func (ucols file))))) (defun create-custom-table-cmd (tablename sql-cmd) "Return SQL command to create a custom table" @@ -159,7 +159,7 @@ (defun create-umls-db-by-insert () "SQL Databases: initializes entire database via SQL insert commands" - (init-umls) + (ensure-init-umls) (init-hash-table) (with-sql-connection (conn) (sql-drop-tables conn) @@ -174,15 +174,14 @@ (copy-cmd #'mysql-copy-cmd)) "SQL Databases: initializes entire database via SQL copy commands. This is much faster that using create-umls-db-insert." - (init-umls) + (ensure-init-umls) (init-hash-table) (translate-all-files extension) (with-sql-connection (conn) (sql-drop-tables conn) (sql-create-tables conn) - (map 'nil - #'(lambda (file) (sql-execute (funcall copy-cmd file extension) conn)) - *umls-files*) + (dolist (file *umls-files*) + (sql-execute (funcall copy-cmd file extension) conn)) (sql-create-indexes conn) (sql-create-custom-tables conn) (sql-create-indexes conn +custom-index-cols+) @@ -213,6 +212,20 @@ This is much faster that using create-umls-db-insert." (translate-line out-ufile line ostream) (princ #\newline ostream))))))) +(defun translate-line (file line strm) + "Translate a single line for sql output" + (flet ((col-value (col value) + (if (eq (datatype col) 'sql-u) + (let ((ui (parse-ui value ""))) + (if (stringp ui) + ui + (write-to-string ui))) + (escape-backslashes value)))) + (print-separated-strings + strm "|" + (mapcar #'col-value (remove-custom-cols (ucols file)) line) + (custom-col-values (custom-ucols-for-file file) line nil)))) + (defun pg-copy-cmd (file extension) "Return postgresql copy statement for a file" (format @@ -226,24 +239,12 @@ This is much faster that using create-umls-db-insert." "LOAD DATA LOCAL INFILE \"~a\" INTO TABLE ~a FIELDS TERMINATED BY \"|\"" (umls-pathname (fil file) extension) (table file))) -(defun col-value (col value) - (if (eq (datatype col) 'sql-u) - (let ((ui (parse-ui value ""))) - (if (stringp ui) - ui - (write-to-string ui))) - (escape-backslashes value))) - -(defun translate-line (file line strm) - "Translate a single line for sql output" - (print-separated-strings - strm "|" - (mapcar #'col-value (remove-custom-cols (ucols file)) line) - (custom-col-values (custom-ucols-for-file file) line nil))) - +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; ;;; Routines for analyzing cost of fixed size storage - +;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defun umls-fixed-size-waste () "Display storage waste if using all fixed size storage" @@ -273,8 +274,7 @@ This is much faster that using create-umls-db-insert." (nreverse avoidable) (nreverse unavoidable)))) (defun display-waste () - (unless *umls-files* - (init-umls)) + (ensure-init-umls) (multiple-value-bind (tw ta tu al ul) (umls-fixed-size-waste) (format t "Total waste: ~d~%" tw) (format t "Total avoidable: ~d~%" ta) @@ -289,24 +289,23 @@ This is much faster that using create-umls-db-insert." (defun max-umls-field () "Return length of longest field" - (unless *umls-files* - (init-umls)) + (declare (optimize (speed 3) (space 0))) + (ensure-init-umls) (let ((max 0)) (declare (fixnum max)) - (dolist (col *umls-cols*) - (when (> (cmax col) max) - (setq max (cmax col)))) + (dolist (ucol *umls-cols*) + (when (> (cmax ucol) max) + (setq max (cmax ucol)))) max)) (defun max-umls-row () "Return length of longest row" - (unless *umls-files* - (init-umls)) + (declare (optimize (speed 3) (space 0))) + (ensure-init-umls) (let ((rowsizes '())) (dolist (file *umls-files*) - (let ((row 0) - (fields (ucols file))) - (dolist (field fields) - (incf row (1+ (cmax field)))) + (let ((row 0)) + (dolist (ucol (ucols file)) + (incf row (1+ (cmax ucol)))) (push row rowsizes))) (car (sort rowsizes #'>)))) diff --git a/data-structures.lisp b/data-structures.lisp index 3500ba4..dfa2845 100644 --- a/data-structures.lisp +++ b/data-structures.lisp @@ -7,7 +7,7 @@ ;;;; Author: Kevin M. Rosenberg ;;;; Date Started: Apr 2000 ;;;; -;;;; $Id: data-structures.lisp,v 1.9 2003/05/06 07:20:31 kevin Exp $ +;;;; $Id: data-structures.lisp,v 1.10 2003/05/07 21:57:06 kevin Exp $ ;;;; ;;;; This file, part of UMLisp, is ;;;; Copyright (c) 2000-2002 by Kevin M. Rosenberg, M.D. @@ -81,7 +81,7 @@ (sqltype :initarg :sqltype :accessor sqltype) (dty :initarg :dty :accessor dty :documentation "new in 2002: suggested SQL datatype") (parse-fun :initarg :parse-fun :accessor parse-fun) - (quotechar :initarg :quotechar :accessor quotechar) + (quote-str :initarg :quote-str :accessor quote-str) (datatype :initarg :datatype :accessor datatype) (custom-value-fun :initarg :custom-value-fun :accessor custom-value-fun)) (:default-initargs :col nil :des nil :ref nil :min nil :av nil :max nil :fil nil diff --git a/parse-2002.lisp b/parse-2002.lisp index 4024eb2..726764f 100644 --- a/parse-2002.lisp +++ b/parse-2002.lisp @@ -8,7 +8,7 @@ ;;;; Author: Kevin M. Rosenberg ;;;; Date Started: Apr 2000 ;;;; -;;;; $Id: parse-2002.lisp,v 1.9 2003/05/06 07:55:15 kevin Exp $ +;;;; $Id: parse-2002.lisp,v 1.10 2003/05/07 21:57:06 kevin Exp $ ;;;; ;;;; This file, part of UMLisp, is ;;;; Copyright (c) 2000-2002 by Kevin M. Rosenberg, M.D. @@ -262,167 +262,66 @@ ;; File & Column functions -(defun init-umls (&optional (alwaysclear nil)) -"Initialize all UMLS file and column structures if not already initialized" - (when (or alwaysclear (null *umls-files*)) - (init-umls-cols) - (init-umls-files) - (init-field-lengths))) +(defun gen-ucols () + (add-ucols (gen-ucols-meta)) + (add-ucols (gen-ucols-custom)) + (add-ucols (gen-ucols-generic "LRFLD")) + (add-ucols (gen-ucols-generic "SRFLD"))) -(defun init-umls-cols () - (setq *umls-cols* (append - (init-meta-cols) - (init-custom-cols) - (init-generic-cols "LRFLD") - (init-generic-cols "SRFLD")))) - -(defun init-meta-cols () +(defun gen-ucols-meta () "Initialize all umls columns" (let ((cols '())) (with-umls-file (line "MRCOLS") (destructuring-bind (col des ref min av max fil dty) line - (let ((c (make-instance 'ucol - :col col - :des des - :ref ref - :min (parse-integer min) - :av (read-from-string av) - :max (parse-integer max) - :fil fil - :dty dty ;; new in 2002 UMLS - :sqltype "VARCHAR" ; default data type - :parse-fun #'add-sql-quotes - :custom-value-fun nil - :quotechar "'"))) - (add-datatype-to-col c (datatype-for-col col)) - (push c cols)))) + (push (make-ucol col des ref (parse-integer min) (read-from-string av) + (parse-integer max) fil dty) + cols))) (nreverse cols))) -(defun init-custom-cols () +(defun gen-ucols-custom () "Initialize umls columns for custom columns" - (let ((cols '())) - (dolist (customcol +custom-cols+) - (let ((c (make-instance 'ucol - :col (nth 1 customcol) :des "" :ref 0 :min 0 :max (nth 3 customcol) - :av 0 :dty nil :fil (nth 0 customcol) :sqltype (nth 2 customcol) - :parse-fun #'add-sql-quotes :custom-value-fun (nth 4 customcol) - :quotechar "'"))) - (add-datatype-to-col c (datatype-for-col (nth 1 customcol))) - (push c cols))) - (nreverse cols))) - -(defun escape-column-name (name) - (substitute #\_ #\/ name)) + (loop for customcol in +custom-cols+ + collect + (make-ucol (nth 1 customcol) "" 0 0 0 (nth 3 customcol) + (nth 0 customcol) nil :sqltype (nth 2 customcol)))) -(defun init-generic-cols (col-filename) +(defun gen-ucols-generic (col-filename) "Initialize for generic (LEX/NET) columns" (let ((cols '())) (with-umls-file (line col-filename) (destructuring-bind (nam des ref fil) line (setq nam (escape-column-name nam)) (dolist (file (delimited-string-to-list fil #\,)) - (let ((c (make-instance 'ucol - :col nam - :des des - :ref ref - :min nil - :av nil - :max nil - :fil file - :dty nil - :sqltype "VARCHAR" ; default data type - :parse-fun #'add-sql-quotes - :custom-value-fun nil - :quotechar "'"))) - (add-datatype-to-col c (datatype-for-col nam)) - (push c cols))))) + (push + (make-ucol nam des ref nil nil nil file nil) + cols)))) (nreverse cols))) -(defun init-umls-files () - (setq *umls-files* (append - (init-generic-files "MRFILES") - (init-generic-files "LRFIL") - (init-generic-files "SRFIL"))) - ;; need to separate this since init-custom-files depends on *umls-files* - (setq *umls-files* (append *umls-files* (init-custom-files)))) +(defun gen-ufiles () + (add-ufiles (gen-ufiles-generic "MRFILES")) + (add-ufiles (gen-ufiles-generic "LRFIL")) + (add-ufiles (gen-ufiles-generic "SRFIL")) + ;; needs to come last + (add-ufiles (gen-ufiles-custom))) -(defun umls-field-string-to-list (fmt) - "Converts a comma delimited list of fields into a list of field names. Will -append a unique number (starting at 2) onto a column name that is repeated in the list" - (let ((field-list (delimited-string-to-list (escape-column-name fmt) #\,)) - (col-count (make-hash-table :test 'equal))) - (dotimes (i (length field-list)) - (declare (fixnum i)) - (let ((col (nth i field-list))) - (multiple-value-bind (key found) (gethash col col-count) - (if found - (let ((next-id (1+ key))) - (setf (nth i field-list) (concatenate 'string col (write-to-string next-id))) - (setf (gethash col col-count) next-id)) - (setf (gethash col col-count) 1))))) - field-list)) - -(defun init-generic-files (files-filename) + +(defun gen-ufiles-generic (files-filename) "Initialize all LEX file structures" (let ((files '())) - (with-umls-file (line files-filename) - (destructuring-bind (fil des fmt cls rws bts) line - (let ((f (make-instance 'ufile - :fil fil - :table (substitute #\_ #\. fil) - :des des - :fmt (escape-column-name fmt) - :cls (parse-integer cls) - :rws (parse-integer rws) - :bts (parse-integer bts) - :fields (concatenate 'list - (umls-field-string-to-list fmt) - (custom-colnames-for-filename fil))))) - (setf (ucols f) (ucols-for-ufile f)) - (push f files)))) - (nreverse files))) - -(defun init-custom-files () - (let ((ffile (make-instance 'ufile - :fil "MRXW.NONENG" :des "Custom NonEnglish Index" :table "MRXW_NONENG" - :cls 5 :rws 0 :bts 0 :fields (fields (find-ufile "MRXW.ENG"))))) - (setf (ucols ffile) - (ucols-for-ufile ffile)) - (list ffile))) - -(defun datatype-for-col (colname) -"Return datatype for column name" - (car (cdr (find colname +col-datatypes+ :key #'car :test #'string-equal)))) + (with-umls-file (line files-filename) + (destructuring-bind (fil des fmt cls rws bts) line + (push (make-ufile + fil des (substitute #\_ #\. fil) (parse-integer cls) + (parse-integer rws) (parse-integer bts) + (concatenate 'list (umls-field-string-to-list fmt) + (custom-colnames-for-filename fil))) + files))) + (nreverse files))) -(defun add-datatype-to-col (col datatype) -"Add data type information to column" - (setf (datatype col) datatype) - (case datatype - (sql-u (setf (sqltype col) "INTEGER" - (parse-fun col) #'parse-ui - (quotechar col) "")) - (sql-s (setf (sqltype col) "SMALLINT" - (parse-fun col) #'parse-integer - (quotechar col) "")) - (sql-l (setf (sqltype col) "BIGINT" - (parse-fun col) #'parse-integer - (quotechar col) "")) - (sql-i (setf (sqltype col) "INTEGER" - (parse-fun col) #'parse-integer - (quotechar col) "")) - (sql-f (setf (sqltype col) "NUMERIC" - (parse-fun col) #'read-from-string - (quotechar col) "")) - (t ; Default column type, optimized text storage - (setf (parse-fun col) #'add-sql-quotes - (quotechar col) "'") - (when (and (cmax col) (av col)) - (if (> (cmax col) 255) - (setf (sqltype col) "TEXT") - (if (< (- (cmax col) (av col)) 4) - (setf (sqltype col) "CHAR") ; if average bytes wasted < 4 - (setf (sqltype col) "VARCHAR"))))))) +(defun gen-ufiles-custom () + (make-ufile "MRXW.NONENG" "Custom NonEnglish Index" "MRXW_NONENG" + 5 0 0 (fields (find-ufile "MRXW.ENG")))) diff --git a/parse-common.lisp b/parse-common.lisp index 07e5cc9..d4f7922 100644 --- a/parse-common.lisp +++ b/parse-common.lisp @@ -7,7 +7,7 @@ ;;;; Author: Kevin M. Rosenberg ;;;; Date Started: Apr 2000 ;;;; -;;;; $Id: parse-common.lisp,v 1.8 2003/05/06 07:44:07 kevin Exp $ +;;;; $Id: parse-common.lisp,v 1.9 2003/05/07 21:57:06 kevin Exp $ ;;;; ;;;; This file, part of UMLisp, is ;;;; Copyright (c) 2000-2002 by Kevin M. Rosenberg, M.D. @@ -21,8 +21,25 @@ (eval-when (:compile-toplevel) (declaim (optimize (speed 3) (safety 1) (compilation-speed 0) (debug 3)))) +(defun ensure-init-umls (&optional (alwaysclear nil)) +"Initialize all UMLS file and column structures if not already initialized" + (when (or alwaysclear (null *umls-files*)) + (gen-ucols) + (gen-ufiles) + (ensure-field-lengths))) + +(defun add-ucols (ucols) + "Adds a ucol or list of ucols to *umls-cols*. Returns input value." + (setq *umls-cols* (append (mklist ucols) *umls-cols*)) + ucols) + +(defun add-ufiles (ufiles) + "Adds a ufile or list of ufiles to *umls-filess*. Returns input value." + (setq *umls-files* (append (mklist ufiles) *umls-files*)) + ufiles) + (defun umls-pathname (filename &optional (extension "")) -"Return pathname for a umls filename" +"Return pathname for a umls filename with an optional extension" (etypecase filename (string (merge-pathnames @@ -48,128 +65,182 @@ ;;; Find field lengths for LEX and NET files -(defun file-field-lengths (files) - (let ((lengths '())) - (dolist (file files) - (setq file (fil file)) - (let (max-field count-field num-fields (count-lines 0)) - (with-umls-file (fields file) - (unless num-fields - (setq num-fields (length fields)) - (setq max-field (make-array num-fields :element-type 'fixnum - :initial-element 0)) - (setq count-field (make-array num-fields :element-type 'number - :initial-element 0))) - (dotimes (i (length fields)) - (declare (fixnum i)) - (let ((len (length (nth i fields)))) - (incf (aref count-field i) len) - (when (> len (aref max-field i)) - (setf (aref max-field i) len)))) - (incf count-lines)) - (dotimes (i num-fields) - (setf (aref count-field i) (float (/ (aref count-field i) count-lines)))) - (push (list file max-field count-field) lengths))) - (nreverse lengths))) - -(defun init-field-lengths () +(defun ensure-field-lengths () "Initial colstruct field lengths for files that don't have a measurement. Currently, these are the LEX and NET files." - (let ((measure-files '())) - (dolist (file *umls-files*) - (let ((filename (fil file))) - (unless (or (char= #\M (char filename 0)) - (char= #\m (char filename 0))) - (push file measure-files)))) - (let ((length-lists (file-field-lengths measure-files))) - (dolist (length-list length-lists) - (let* ((filename (car length-list)) - (max-field (cadr length-list)) - (av-field (caddr length-list)) - (file (find-ufile filename))) - (when file - (if (/= (length max-field) (length (fields file))) - (format t "Warning: Number of file fields ~A doesn't match length of fields in file structure ~S" - max-field file) - (dotimes (i (max (length max-field) (length (fields file)))) - (declare (fixnum i)) - (let* ((field (nth i (fields file))) - (col (find-ucol field filename))) - (if col - (progn - (setf (cmax col) (aref max-field i)) - (setf (av col) (aref av-field i)) - (add-datatype-to-col col (datatype-for-col (col col)))) - (error "can't find column ~A" field))))))))))) + (dolist (length-list (ufiles-field-lengths (ufiles-to-measure))) + (destructuring-bind (filename fields-max fields-av) length-list + (let ((file (find-ufile filename))) + (unless file + (error "Can't find ~A filename in ufiles")) + (unless (= (length fields-max) (length (fields file))) + (error + "Number of file fields ~A not equal to field count in ufile ~S" + fields-max file)) + (dotimes (i (length (fields file))) + (declare (fixnum i)) + (let* ((field (nth i (fields file))) + (col (find-ucol field filename))) + (unless col + (error "can't find column ~A" field)) + (setf (cmax col) (aref fields-max i)) + (setf (av col) (aref fields-av i)) + (ensure-ucol-datatype col (datatype-for-colname (col col))))))))) - +(defun ufiles-to-measure () + "Returns a list of ufiles to measure" + (loop for ufile in *umls-files* + unless (or (char= #\M (char (fil ufile) 0)) + (char= #\m (char (fil ufile) 0))) + collect ufile)) + + +(defun ufiles-field-lengths (ufiles) + "Returns a list of lists of containing (FILE MAX AV)" + (loop for ufile in ufiles collect (file-field-lengths (fil ufile)))) + +(defun file-field-lengths (filename) + "Returns a list of FILENAME MAX AV" + (let (fields-max fields-av num-fields (count-lines 0)) + (with-umls-file (line filename) + (unless num-fields + (setq num-fields (length line)) + (setq fields-max (make-array num-fields :element-type 'fixnum + :initial-element 0)) + (setq fields-av (make-array num-fields :element-type 'number + :initial-element 0))) + (dotimes (i num-fields) + (declare (fixnum i)) + (let ((len (length (nth i line)))) + (incf (aref fields-av i) len) + (when (> len (aref fields-max i)) + (setf (aref fields-max i) len)))) + (incf count-lines)) + (dotimes (i num-fields) + (setf (aref fields-av i) (float (/ (aref fields-av i) count-lines)))) + (list filename fields-max fields-av))) ;;; UMLS column/file functions -(defun find-col-in-columns (colname filename cols) +(defun find-ucol-of-colname (colname filename ucols) "Returns list of umls-col structure for a column name and a filename" - (dolist (col cols) - (when (and (string-equal filename (fil col)) - (string-equal colname (col col))) - (return-from find-col-in-columns col))) - nil) - -(defun find-or-make-col-in-columns (colname filename cols) - (let ((col (find-col-in-columns colname filename cols))) - (if col - col - ;; try to find column name without a terminal digit - (let* ((last-char (char colname (1- (length colname)))) - (digit (- (char-code last-char) (char-code #\0)))) - (if (and (>= digit 0) (<= digit 9)) - (let ((base-colname (subseq colname 0 (1- (length colname))))) - (setq col (find-col-in-columns base-colname filename cols)) - (if col - (let ((new-col (make-instance 'ucol - :col (copy-seq colname) - :des (copy-seq (des col)) - :ref (copy-seq (ref col)) - :min (cmin col) - :max (cmax col) - :fil (copy-seq (fil col)) - :sqltype (copy-seq (sqltype col)) - :dty (copy-seq (dty col)) - :parse-fun (parse-fun col) - :quotechar (copy-seq (quotechar col)) - :datatype (datatype col) - :custom-value-fun (custom-value-fun col)))) - (push new-col *umls-cols*) - new-col) - (error "Couldn't find a base column for col ~A in file ~A" - colname filename))) - (let ((new-col (make-instance 'ucol - :col (copy-seq colname) - :des "Unknown" - :ref "" - :min nil - :max nil - :fil filename - :sqltype "VARCHAR" - :dty nil - :parse-fun #'add-sql-quotes - :quotechar "'" - :datatype nil - :custom-value-fun nil))) - (push new-col *umls-cols*) - new-col)))))) + (dolist (ucol ucols nil) + (when (and (string-equal filename (fil ucol)) + (string-equal colname (col ucol))) + (return-from find-ucol-of-colname ucol)))) + +(defun ensure-col-in-columns (colname filename ucols) + (aif (find-ucol-of-colname colname filename ucols) + it + (add-ucols (make-ucol-for-column colname filename ucols)))) + +(defun make-ucol-for-column (colname filename ucols) + ;; try to find column name without a terminal digit + (let* ((len (length colname)) + (last-digit? (digit-char-p (char colname (1- len)))) + (base-colname (if last-digit? + (subseq colname 0 (1- len)) + colname)) + (ucol (when last-digit? + (find-ucol-of-colname base-colname filename ucols)))) + (when (and last-digit? (null ucol)) + (error "Couldn't find a base column for col ~A in file ~A" + colname filename)) + (copy-or-new-ucol colname filename ucol))) + +(defun copy-or-new-ucol (colname filename ucol) + (if ucol + (make-instance + 'ucol + :col (copy-seq colname) :des (copy-seq (des ucol)) :ref (copy-seq (ref ucol)) + :min (cmin ucol) :max (cmax ucol) :fil (copy-seq (fil ucol)) + :sqltype (copy-seq (sqltype ucol)) :dty (copy-seq (dty ucol)) + :parse-fun (parse-fun ucol) :quote-str (copy-seq (quote-str ucol)) + :datatype (datatype ucol) :custom-value-fun (custom-value-fun ucol)) + (make-empty-ucol colname filename))) + +(defun make-ucol (col des ref min av max fil dty + &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes) + (quote-str "'") (custom-value-fun)) + (let ((ucol (make-instance + 'ucol + :col col :des des :ref ref :min min :av av :max max :fil fil + :dty dty :sqltype sqltype :parse-fun parse-fun + :quote-str quote-str :custom-value-fun custom-value-fun))) + (ensure-ucol-datatype ucol (datatype-for-colname col)) + ucol)) + +(defun make-empty-ucol (colname filename) + (make-ucol (copy-seq colname) "Unknown" "" nil nil nil filename nil)) (defun find-ucol (colname filename) "Returns list of umls-col structure for a column name and a filename" - (find-or-make-col-in-columns colname filename *umls-cols*)) + (ensure-col-in-columns colname filename *umls-cols*)) (defun find-ufile (filename) "Returns umls-file structure for a filename" - (find-if (lambda (f) (string-equal filename (fil f))) *umls-files*)) - -(defun ucols-for-ufile (file) - "Returns list of umls-cols for a file structure" - (let ((filename (fil file))) - (mapcar (lambda (col) (find-ucol col filename)) - (fields file)))) - - + (find-if #'(lambda (f) (string-equal filename (fil f))) *umls-files*)) + +(defun find-ucols-for-filename (filename) + "Returns list of umls-cols for a file structure" + (loop for colname in (fields (find-ufile filename)) + collect (find-ucol colname filename))) + +(defun umls-field-string-to-list (fmt) + "Converts a comma delimited list of fields into a list of field names. Will +append a unique number (starting at 2) onto a column name that is repeated in the list" + (let ((col-counts (make-hash-table :test 'equal))) + (loop for colname in (delimited-string-to-list (escape-column-name fmt) #\,) + collect + (multiple-value-bind (value found) (gethash col col-counts) + (cond + (found + (incf (gethash col col-counts)) + (concatenate 'string colname (write-to-string (1+ value)))) + (t + (setf (gethash col col-counts) 1) + colname)))))) + +(defun make-ufile (fil des table cls rws bts fields) + (let ((ufile + (make-instance + 'ufile :fil fil :des des :table table :cls cls :rws rws :bts bts + :fields fields))) + (setf (ucols ufile) (find-ucols-for-filename fil)) + ufile)) + +(defun datatype-for-colname (colname) +"Return datatype for column name" + (second (find colname +col-datatypes+ :key #'car :test #'string-equal))) + +(defun ensure-ucol-datatype (col datatype) +"Add data type information to column" + (setf (datatype col) datatype) + (case datatype + (sql-u (setf (sqltype col) "INTEGER" + (parse-fun col) #'parse-ui + (quote-str col) "")) + (sql-s (setf (sqltype col) "SMALLINT" + (parse-fun col) #'parse-integer + (quote-str col) "")) + (sql-l (setf (sqltype col) "BIGINT" + (parse-fun col) #'parse-integer + (quote-str col) "")) + (sql-i (setf (sqltype col) "INTEGER" + (parse-fun col) #'parse-integer + (quote-str col) "")) + (sql-f (setf (sqltype col) "NUMERIC" + (parse-fun col) #'read-from-string + (quote-str col) "")) + (t ; Default column type, optimized text storage + (setf (parse-fun col) #'add-sql-quotes + (quote-str col) "'") + (when (and (cmax col) (av col)) + (if (> (cmax col) 255) + (setf (sqltype col) "TEXT") + (if (< (- (cmax col) (av col)) 4) + (setf (sqltype col) "CHAR") ; if average bytes wasted < 4 + (setf (sqltype col) "VARCHAR"))))))) + +(defun escape-column-name (name) + (substitute #\_ #\/ name)) diff --git a/parse-macros.lisp b/parse-macros.lisp index fbc2c24..cd7e501 100644 --- a/parse-macros.lisp +++ b/parse-macros.lisp @@ -2,12 +2,12 @@ ;;;; ************************************************************************* ;;;; FILE IDENTIFICATION ;;;; -;;;; Name: parse-macros.lisp -;;;; Purpose: Macros for UMLS file parsing -;;;; Programmer: Kevin M. Rosenberg -;;;; Date Started: Apr 2000 +;;;; Name: parse-macros.lisp +;;;; Purpose: Macros for UMLS file parsing +;;;; Author: Kevin M. Rosenberg +;;;; Date Started: Apr 2000 ;;;; -;;;; $Id: parse-macros.lisp,v 1.6 2003/05/06 06:09:29 kevin Exp $ +;;;; $Id: parse-macros.lisp,v 1.7 2003/05/07 21:57:06 kevin Exp $ ;;;; ;;;; This file, part of UMLisp, is ;;;; Copyright (c) 2000-2002 by Kevin M. Rosenberg, M.D. diff --git a/run-tests.lisp b/run-tests.lisp new file mode 100644 index 0000000..c9bbd31 --- /dev/null +++ b/run-tests.lisp @@ -0,0 +1,27 @@ +(defpackage #:run-tests (:use #:cl)) +(in-package #:run-tests) + +(require 'rt) +(require 'kmrcl) +(require 'clsql-mysql) +(require 'clsql) +(require 'hyperobject) +(load "umlisp.asd") +(asdf:oos 'asdf:test-op 'umlisp) + +(defun quit (&optional (code 0)) + "Function to exit the Lisp implementation. Copied from CLOCC's QUIT function." + #+allegro (excl:exit code) + #+clisp (#+lisp=cl ext:quit #-lisp=cl lisp:quit code) + #+(or cmu scl) (ext:quit code) + #+cormanlisp (win32:exitprocess code) + #+gcl (lisp:bye code) + #+lispworks (lw:quit :status code) + #+lucid (lcl:quit code) + #+sbcl (sb-ext:quit :unix-status (typecase code (number code) (null 0) (t 1))) + #+mcl (ccl:quit code) + #-(or allegro clisp cmu scl cormanlisp gcl lispworks lucid sbcl mcl) + (error 'not-implemented :proc (list 'quit code))) + +(quit) + diff --git a/umlisp.asd b/umlisp.asd index 1c8cb6a..2d81e58 100644 --- a/umlisp.asd +++ b/umlisp.asd @@ -7,7 +7,7 @@ ;;;; Author: Kevin M. Rosenberg ;;;; Date Started: Apr 2000 ;;;; -;;;; $Id: umlisp.asd,v 1.16 2003/05/03 17:10:08 kevin Exp $ +;;;; $Id: umlisp.asd,v 1.17 2003/05/07 21:57:06 kevin Exp $ ;;;; ;;;; This file, part of UMLisp, is ;;;; Copyright (c) 2000-2002 by Kevin M. Rosenberg, M.D. @@ -29,7 +29,7 @@ (:file "parse-macros" :depends-on ("sql")) (:file "parse-2002" :depends-on ("parse-macros")) (:file "parse-common" :depends-on ("parse-2002")) - (:file "sql-create" :depends-on ("parse-common")) + (:file "create-sql" :depends-on ("parse-common")) (:file "classes" :depends-on ("utils")) (:file "class-support" :depends-on ("classes")) (:file "sql-classes" :depends-on ("class-support" "sql")) -- 2.34.1