X-Git-Url: http://git.kpe.io/?p=umlisp.git;a=blobdiff_plain;f=parse-common.lisp;h=b1395665b68eb612ef4de7fcfccaa16436ce2981;hp=c9adcf740cd7d856a383723af8a99f55922cbf42;hb=HEAD;hpb=08af5459d3ba7d229a8339d242ef742cfe846861 diff --git a/parse-common.lisp b/parse-common.lisp index c9adcf7..f7fab1e 100644 --- a/parse-common.lisp +++ b/parse-common.lisp @@ -7,10 +7,8 @@ ;;;; Author: Kevin M. Rosenberg ;;;; Created: Apr 2000 ;;;; -;;;; $Id$ -;;;; ;;;; This file, part of UMLisp, is -;;;; Copyright (c) 2000-2006 by Kevin M. Rosenberg, M.D. +;;;; Copyright (c) 2000-2010 by Kevin M. Rosenberg, M.D. ;;;; ;;;; UMLisp users are granted the rights to distribute and use this software ;;;; as governed by the terms of the GNU General Public License. @@ -23,12 +21,14 @@ (handler-case (when (or alwaysclear (null *umls-files*)) (setf *umls-cols* nil *umls-files* nil) - (gen-ucols) (gen-ufiles) + (gen-ucols) + (set-ucols-for-ufiles *umls-files*) (ensure-field-lengths)) (error (e) + (warn "Error reading ucols+ufiles: ~A." e) (setf *umls-cols* nil *umls-files* nil) - (error e))) + nil)) t) @@ -45,8 +45,8 @@ (defun ufile-pathname (ufile &optional (extension "")) "Return pathname for a umls filename with an optional extension" (assert (typep ufile 'ufile)) - (let* ((dirs (append (list (dir ufile)) - (awhen (subdir ufile) (list it)))) + (let* ((dirs (nconc (list (dir ufile)) + (awhen (subdir ufile) (list it)))) (name-list (delimited-string-to-list (fil ufile) #\.)) (name (if (second name-list) (first name-list) @@ -55,7 +55,7 @@ (concatenate 'string (second name-list) (or extension ""))))) (merge-pathnames (make-pathname :name name :type type - :directory (cons :relative dirs)) + :directory (cons :relative dirs)) *umls-path*))) (defun umls-pathname (filename &optional (extension "")) @@ -89,29 +89,32 @@ "Initial colstruct field lengths for files that don't have a measurement. Currently, these are the LEX and NET files." (dolist (length-list (ufiles-field-lengths (ufiles-to-measure))) - (destructuring-bind (filename fields-max fields-av) length-list + (destructuring-bind (filename fields-max fields-av count-lines) length-list (let ((file (find-ufile filename))) - (unless file - (error "Can't find ~A filename in ufiles" filename)) - (unless (= (length fields-max) (length (fields file))) - (error - "Number of file fields ~A not equal to field count in ufile ~S" - fields-max file)) - (dotimes (i (length (fields file))) - (declare (fixnum i)) - (let* ((field (nth i (fields file))) - (col (find-ucol field filename))) - (unless col - (error "can't find column ~A" field)) - (setf (cmax col) (aref fields-max i)) - (setf (av col) (aref fields-av i)) - (ensure-ucol-datatype col (datatype-for-colname (col col))))))))) + (unless file + (error "Can't find ~A filename in ufiles" filename)) + (if (zerop count-lines) + (warn "File ~A is empty." filename) + (progn + (unless (= (length fields-max) (length (fields file))) + (error + "Number of file fields ~A not equal to field count in ufile ~S" + fields-max file)) + (dotimes (i (length (fields file))) + (declare (fixnum i)) + (let* ((field (nth i (fields file))) + (col (find-ucol field filename))) + (unless col + (error "can't find column ~A" field)) + (setf (cmax col) (aref fields-max i)) + (setf (av col) (aref fields-av i)) + (ensure-ucol-datatype col (datatype-for-colname (col col))))))))))) (defun ufiles-to-measure () "Returns a list of ufiles to measure" (loop for ufile in *umls-files* - unless (or (char= #\M (schar (fil ufile) 0)) - (char= #\m (schar (fil ufile) 0))) + unless (or (char= #\M (schar (fil ufile) 0)) + (char= #\m (schar (fil ufile) 0))) collect ufile)) @@ -123,28 +126,34 @@ Currently, these are the LEX and NET files." "Returns a list of FILENAME MAX AV" (declare (optimize (speed 3) (safety 0))) (let (fields-max fields-av num-fields (count-lines 0)) + (declare (fixnum count-lines)) (with-umls-ufile (line ufile) (unless num-fields - (setq num-fields (length line)) - (setq fields-max (make-array num-fields :element-type 'fixnum - :initial-element 0)) - (setq fields-av (make-array num-fields :element-type 'number - :initial-element 0))) - (dotimes (i num-fields) - (declare (fixnum i)) - (let* ((str (nth i line)) + (setq num-fields (length line)) + (setq fields-max (make-array num-fields :element-type 'fixnum + :initial-element 0)) + (setq fields-av (make-array num-fields :element-type '(or integer float) + :initial-element 0))) + (dotimes (i (or num-fields 0)) + (declare (fixnum i)) + (let* ((str (nth i line)) (len (length #-(and clisp unicode) str #+(and clisp unicode) (if *octet-sql-storage* (ext:convert-string-to-bytes str charset:utf-8) str)))) - (incf (aref fields-av i) len) - (when (> len (aref fields-max i)) - (setf (aref fields-max i) len)))) + #-(and clisp unicode) (declare (string str)) + (declare (type (integer 0 100000000) len)) + (incf (aref fields-av i) len) + (when (> len (aref fields-max i)) + (setf (aref fields-max i) len)))) (incf count-lines)) - (dotimes (i num-fields) - (setf (aref fields-av i) (float (/ (aref fields-av i) count-lines)))) - (list (fil ufile) fields-max fields-av))) + (dotimes (i (or num-fields 0)) + (setf (aref fields-av i) + (if (plusp count-lines) + (float (/ (aref fields-av i) count-lines)) + 0))) + (list (fil ufile) fields-max fields-av count-lines))) ;;; UMLS column/file functions @@ -152,7 +161,7 @@ Currently, these are the LEX and NET files." "Returns list of umls-col structure for a column name and a filename" (dolist (ucol ucols nil) (when (and (string-equal filename (fil ucol)) - (string-equal colname (col ucol))) + (string-equal colname (col ucol))) (return-from find-ucol-of-colname ucol)))) (defun ensure-col-in-columns (colname filename ucols) @@ -163,15 +172,15 @@ Currently, these are the LEX and NET files." (defun make-ucol-for-column (colname filename ucols) ;; try to find column name without a terminal digit (let* ((len (length colname)) - (last-digit? (digit-char-p (schar colname (1- len)))) - (base-colname (if last-digit? - (subseq colname 0 (1- len)) - colname)) - (ucol (when last-digit? - (find-ucol-of-colname base-colname filename ucols)))) + (last-digit? (digit-char-p (schar colname (1- len)))) + (base-colname (if last-digit? + (subseq colname 0 (1- len)) + colname)) + (ucol (when last-digit? + (find-ucol-of-colname base-colname filename ucols)))) (when (and last-digit? (null ucol)) (error "Couldn't find a base column for col ~A in file ~A" - colname filename)) + colname filename)) (copy-or-new-ucol colname filename ucol))) (defun copy-or-new-ucol (colname filename ucol) @@ -192,24 +201,24 @@ Currently, these are the LEX and NET files." nil) (function (if (compiled-function-p fun) - fun - (compile nil fun))) + fun + (compile nil fun))) (list (compile nil fun)))) (defun make-ucol (col des ref min av max fil dty - &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes) - (quote-str "'") (custom-value-fun)) + &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes) + (quote-str "'") (custom-value-fun)) (let ((ucol (make-instance - 'ucol - :col col :des des :ref ref :min min :av av - :max (if (eql max 0) 1 max) ;; ensure at least one char wide - :fil fil - :dty dty - :sqltype sqltype - :quote-str quote-str - :parse-fun (ensure-compiled-fun parse-fun) - :custom-value-fun (ensure-compiled-fun custom-value-fun)))) + 'ucol + :col col :des des :ref ref :min min :av av + :max (if (eql max 0) 1 max) ;; ensure at least one char wide + :fil fil + :dty dty + :sqltype sqltype + :quote-str quote-str + :parse-fun (ensure-compiled-fun parse-fun) + :custom-value-fun (ensure-compiled-fun custom-value-fun)))) (ensure-ucol-datatype ucol (datatype-for-colname col)) ucol)) @@ -223,37 +232,49 @@ Currently, these are the LEX and NET files." (defun find-ufile (filename) "Returns umls-file structure for a filename" - (find-if #'(lambda (f) (string-equal filename (fil f))) *umls-files*)) + (find-if #'(lambda (f) (string= filename (fil f))) *umls-files*)) + +(defun position-field-file (filename fieldname) + "Returns the position of a field in a file" + (let ((ufile (find-ufile filename))) + (unless ufile + (warn "Unable to find ufile for filename ~A." filename) + (return-from position-field-file nil)) + (let ((pos (position fieldname (fields ufile) :test #'string=))) + (unless pos + (warn "Unable to find field ~A in ufile ~S." fieldname ufile) + (return-from position-field-file nil)) + pos))) (defun find-ucols-for-ufile (ufile) "Returns list of umls-cols for a file structure" (loop for colname in (fields ufile) collect (find-ucol colname - (if (subdir ufile) - (concatenate 'string (subdir ufile) "/" (fil ufile)) - (fil ufile))))) + (if (subdir ufile) + (concatenate 'string (subdir ufile) "/" (fil ufile)) + (fil ufile))))) (defun umls-field-string-to-list (fmt) "Converts a comma delimited list of fields into a list of field names. Will append a unique number (starting at 2) onto a column name that is repeated in the list" (let ((col-counts (make-hash-table :test 'equal))) (loop for colname in (delimited-string-to-list (escape-column-name fmt) #\,) - collect - (multiple-value-bind (value found) (gethash colname col-counts) - (cond - (found - (incf (gethash colname col-counts)) - (concatenate 'string colname (write-to-string (1+ value)))) - (t - (setf (gethash colname col-counts) 1) - colname)))))) + collect + (multiple-value-bind (value found) (gethash colname col-counts) + (cond + (found + (incf (gethash colname col-counts)) + (concatenate 'string colname (write-to-string (1+ value)))) + (t + (setf (gethash colname col-counts) 1) + colname)))))) (defun decompose-fil (fil) (if fil (let ((pos (position #\/ fil))) - (if pos - (values (subseq fil (1+ pos)) (subseq fil 0 pos)) - (values fil nil))) + (if pos + (values (subseq fil (1+ pos)) (subseq fil 0 pos)) + (values fil nil))) (values nil nil))) (defun filename-to-tablename (file) @@ -265,18 +286,27 @@ append a unique number (starting at 2) onto a column name that is repeated in th (defun make-ufile (dir fil des cls rws bts fields) (multiple-value-bind (file subdir) (decompose-fil fil) (let ((ufile (make-instance 'ufile :dir dir :fil file :subdir subdir - :des des :cls cls - :rws rws :bts bts :fields fields - :table (filename-to-tablename file)))) - (setf (ucols ufile) (find-ucols-for-ufile ufile)) + :des des :cls cls + :rws rws :bts bts :fields fields + :table (filename-to-tablename file)))) ufile))) +(defun set-ucols-for-ufiles (ufiles) + (dolist (ufile ufiles) + (setf (ucols ufile) (find-ucols-for-ufile ufile)))) + (defun datatype-for-colname (colname) "Return datatype for column name" (second (find colname +col-datatypes+ :key #'car :test #'string-equal))) (defun canonicalize-column-type (type) (cond + ((string-equal type "TINYINT") + (case *umls-sql-type* + (:mysql "TINYINT") + ((:postgresql :postgresql-socket) "INT1") + (:oracle "NUMBER(3,0)") + (t "INTEGER"))) ((string-equal type "SMALLINT") (case *umls-sql-type* (:mysql "SMALLINT") @@ -321,27 +351,44 @@ append a unique number (starting at 2) onto a column name that is repeated in th (setf (datatype col) datatype) (case datatype (sql-u (setf (sqltype col) (canonicalize-column-type "INTEGER") - (parse-fun col) #'parse-ui - (quote-str col) "")) + (parse-fun col) #'parse-ui + (quote-str col) "")) (sql-s (setf (sqltype col) (canonicalize-column-type "SMALLINT") - (parse-fun col) #'parse-integer - (quote-str col) "")) + (parse-fun col) #'parse-integer + (quote-str col) "")) (sql-l (setf (sqltype col) (canonicalize-column-type "BIGINT") - (parse-fun col) #'parse-integer - (quote-str col) "")) + (parse-fun col) #'parse-integer + (quote-str col) "")) (sql-i (setf (sqltype col) (canonicalize-column-type "INTEGER") - (parse-fun col) #'parse-integer - (quote-str col) "")) + (parse-fun col) #'parse-integer + (quote-str col) "")) + (sql-t (setf (sqltype col) (canonicalize-column-type "TINYINT") + (parse-fun col) #'parse-integer + (quote-str col) "")) (sql-f (setf (sqltype col) (canonicalize-column-type "NUMERIC") - (parse-fun col) #'read-from-string - (quote-str col) "")) - (t ; Default column type, optimized text storage + (parse-fun col) #'read-from-string + (quote-str col) "")) + (t ; Default column type, optimized text storage (setf (parse-fun col) #'add-sql-quotes - (quote-str col) "'") + (quote-str col) "'") (when (and (cmax col) (av col)) (if (> (cmax col) 255) - (setf (sqltype col) (canonicalize-column-type "TEXT")) - (setf (sqltype col) (canonicalize-column-type "VARCHAR"))))))) + (setf (sqltype col) (canonicalize-column-type "TEXT")) + (setf (sqltype col) (canonicalize-column-type "VARCHAR"))))))) (defun escape-column-name (name) (substitute #\_ #\/ name)) + +;; SQLNAME is required for collision of SQL reserved words (MYSQL 8: RANK) +;; and column names in UMLS (RANK in MRRANK) +(defvar *sql-reserved-names* '("RANK")) +(defmethod sqlname ((c ucol)) + (sqlname (col c))) +(defmethod sqlname ((name string)) + (if (find name *sql-reserved-names* :test #'string-equal) + (concatenate 'string "_" name) + name)) +(defmethod sqlname ((l list)) + (mapcar #'sqlname l)) +(defmethod sqlname ((s symbol)) + (sqlname (symbol-name s)))