;;;; *************************************************************************
;;;; FILE IDENTIFICATION
;;;;
-;;;; Name: parse-common.lisp
-;;;; Purpose: Common, stable parsing routines for UMLisp
-;;;; Programmer: Kevin M. Rosenberg
-;;;; Date Started: Apr 2000
-;;;;
-;;;; $Id: parse-common.lisp,v 1.3 2002/10/18 03:57:39 kevin Exp $
+;;;; Name: parse-common.lisp
+;;;; Purpose: Common, stable parsing routines for UMLisp
+;;;; Author: Kevin M. Rosenberg
+;;;; Created: Apr 2000
;;;;
;;;; This file, part of UMLisp, is
-;;;; Copyright (c) 2000-2002 by Kevin M. Rosenberg, M.D.
+;;;; Copyright (c) 2000-2010 by Kevin M. Rosenberg, M.D.
;;;;
;;;; UMLisp users are granted the rights to distribute and use this software
;;;; as governed by the terms of the GNU General Public License.
;;;; *************************************************************************
-(in-package :umlisp)
-(declaim (optimize (speed 3) (safety 1) (compilation-speed 0) (debug 3)))
+(in-package #:umlisp)
+
+(defun ensure-ucols+ufiles (&optional (alwaysclear nil))
+ "Initialize all UMLS file and column structures if not already initialized"
+ (handler-case
+ (when (or alwaysclear (null *umls-files*))
+ (setf *umls-cols* nil *umls-files* nil)
+ (gen-ufiles)
+ (gen-ucols)
+ (set-ucols-for-ufiles *umls-files*)
+ (ensure-field-lengths))
+ (error (e)
+ (warn "Error reading ucols+ufiles: ~A." e)
+ (setf *umls-cols* nil *umls-files* nil)
+ nil))
+ t)
+
+
+(defun add-ucols (ucols)
+ "Adds a ucol or list of ucols to *umls-cols*. Returns input value."
+ (setq *umls-cols* (append (mklist ucols) *umls-cols*))
+ ucols)
+
+(defun add-ufiles (ufiles)
+ "Adds a ufile or list of ufiles to *umls-filess*. Returns input value."
+ (setq *umls-files* (append (mklist ufiles) *umls-files*))
+ ufiles)
+
+(defun ufile-pathname (ufile &optional (extension ""))
+ "Return pathname for a umls filename with an optional extension"
+ (assert (typep ufile 'ufile))
+ (let* ((dirs (nconc (list (dir ufile))
+ (awhen (subdir ufile) (list it))))
+ (name-list (delimited-string-to-list (fil ufile) #\.))
+ (name (if (second name-list)
+ (first name-list)
+ (concatenate 'string (first name-list) (or extension ""))))
+ (type (when (second name-list)
+ (concatenate 'string (second name-list) (or extension "")))))
+ (merge-pathnames
+ (make-pathname :name name :type type
+ :directory (cons :relative dirs))
+ *umls-path*)))
(defun umls-pathname (filename &optional (extension ""))
-"Return pathname for a umls filename"
+"Return pathname for a umls filename with an optional extension"
(etypecase filename
(string
- (merge-pathnames
- (make-pathname :name (concatenate 'string filename extension))
- (case (char filename 0)
- ((#\M #\m)
- *meta-path*)
- ((#\L #\l)
- *lex-path*)
- ((#\S #\s)
- *net-path*)
- (t
- *umls-path*))))
+ (let* ((name-list (delimited-string-to-list filename #\.))
+ (name (if (second name-list)
+ (first name-list)
+ (concatenate 'string (first name-list) (or extension ""))))
+ (type (when (second name-list)
+ (concatenate 'string (second name-list) (or extension "")))))
+ (merge-pathnames
+ (make-pathname :name name :type type)
+ (case (schar filename 0)
+ ((#\M #\m)
+ *meta-path*)
+ ((#\L #\l)
+ *lex-path*)
+ ((#\S #\s)
+ *net-path*)
+ (t
+ *umls-path*)))))
(pathname
- filename)))
-
-(defun read-umls-line (strm)
- "Read a line from a UMLS stream, split into fields"
- (let ((line (read-line strm nil 'eof)))
- (if (stringp line) ;; ensure not 'eof
- (let* ((len (length line))
- (maybe-remove-terminal ;; LRWD doesn't have '|' at end of line
- (if (char= #\| (char line (1- len)))
- (subseq line 0 (1- len))
- line)))
- (declare (fixnum len))
- (delimited-string-to-list maybe-remove-terminal #\|))
- line)))
+ filename)))
;;; Find field lengths for LEX and NET files
-(defun file-field-lengths (files)
- (let ((lengths '()))
- (dolist (file files)
- (setq file (umls-file-fil file))
- (let (max-field count-field num-fields (count-lines 0))
- (with-umls-file (fields file)
- (unless num-fields
- (setq num-fields (length fields))
- (setq max-field (make-array num-fields :element-type 'fixnum
- :initial-element 0))
- (setq count-field (make-array num-fields :element-type 'number
- :initial-element 0)))
- (dotimes (i (length fields))
- (declare (fixnum i))
- (let ((len (length (nth i fields))))
- (incf (aref count-field i) len)
- (when (> len (aref max-field i))
- (setf (aref max-field i) len))))
- (incf count-lines))
- (dotimes (i num-fields)
- (setf (aref count-field i) (float (/ (aref count-field i) count-lines))))
- (push (list file max-field count-field) lengths)))
- (nreverse lengths)))
-
-(defun init-field-lengths ()
+(defun ensure-field-lengths ()
"Initial colstruct field lengths for files that don't have a measurement.
Currently, these are the LEX and NET files."
- (let ((measure-files '()))
- (dolist (file *umls-files*)
- (let ((filename (umls-file-fil file)))
- (unless (or (char= #\M (char filename 0))
- (char= #\m (char filename 0)))
- (push file measure-files))))
- (let ((length-lists (file-field-lengths measure-files)))
- (dolist (length-list length-lists)
- (let* ((filename (car length-list))
- (max-field (cadr length-list))
- (av-field (caddr length-list))
- (file (find-umls-file filename)))
- (when file
- (if (/= (length max-field) (length (umls-file-fields file)))
- (format t "Warning: Number of file fields ~A doesn't match length of fields in file structure ~S"
- max-field file)
- (dotimes (i (max (length max-field) (length (umls-file-fields file))))
- (declare (fixnum i))
- (let* ((field (nth i (umls-file-fields file)))
- (col (find-umls-col field filename)))
- (if col
- (progn
- (setf (umls-col-max col) (aref max-field i))
- (setf (umls-col-av col) (aref av-field i))
- (add-datatype-to-col col (datatype-for-col (umls-col-col col))))
- (error "can't find column ~A" field)))))))))))
-
+ (dolist (length-list (ufiles-field-lengths (ufiles-to-measure)))
+ (destructuring-bind (filename fields-max fields-av count-lines) length-list
+ (let ((file (find-ufile filename)))
+ (unless file
+ (error "Can't find ~A filename in ufiles" filename))
+ (if (zerop count-lines)
+ (warn "File ~A is empty." filename)
+ (progn
+ (unless (= (length fields-max) (length (fields file)))
+ (error
+ "Number of file fields ~A not equal to field count in ufile ~S"
+ fields-max file))
+ (dotimes (i (length (fields file)))
+ (declare (fixnum i))
+ (let* ((field (nth i (fields file)))
+ (col (find-ucol field filename)))
+ (unless col
+ (error "can't find column ~A" field))
+ (setf (cmax col) (aref fields-max i))
+ (setf (av col) (aref fields-av i))
+ (ensure-ucol-datatype col (datatype-for-colname (col col)))))))))))
+
+(defun ufiles-to-measure ()
+ "Returns a list of ufiles to measure"
+ (loop for ufile in *umls-files*
+ unless (or (char= #\M (schar (fil ufile) 0))
+ (char= #\m (schar (fil ufile) 0)))
+ collect ufile))
+
+(defun ufiles-field-lengths (ufiles)
+ "Returns a list of lists of containing (FILE MAX AV)"
+ (loop for ufile in ufiles collect (file-field-lengths ufile)))
+
+(defun file-field-lengths (ufile)
+ "Returns a list of FILENAME MAX AV"
+ (declare (optimize (speed 3) (safety 0)))
+ (let (fields-max fields-av num-fields (count-lines 0))
+ (declare (fixnum count-lines))
+ (with-umls-ufile (line ufile)
+ (unless num-fields
+ (setq num-fields (length line))
+ (setq fields-max (make-array num-fields :element-type 'fixnum
+ :initial-element 0))
+ (setq fields-av (make-array num-fields :element-type '(or integer float)
+ :initial-element 0)))
+ (dotimes (i (or num-fields 0))
+ (declare (fixnum i))
+ (let* ((str (nth i line))
+ (len (length #-(and clisp unicode) str
+ #+(and clisp unicode)
+ (if *octet-sql-storage*
+ (ext:convert-string-to-bytes str charset:utf-8)
+ str))))
+ #-(and clisp unicode) (declare (string str))
+ (declare (type (integer 0 100000000) len))
+ (incf (aref fields-av i) len)
+ (when (> len (aref fields-max i))
+ (setf (aref fields-max i) len))))
+ (incf count-lines))
+ (dotimes (i (or num-fields 0))
+ (setf (aref fields-av i)
+ (if (plusp count-lines)
+ (float (/ (aref fields-av i) count-lines))
+ 0)))
+ (list (fil ufile) fields-max fields-av count-lines)))
;;; UMLS column/file functions
-(defun find-col-in-columns (colname filename cols)
+(defun find-ucol-of-colname (colname filename ucols)
"Returns list of umls-col structure for a column name and a filename"
- (dolist (col cols)
- (when (and (string-equal filename (umls-col-fil col))
- (string-equal colname (umls-col-col col)))
- (return-from find-col-in-columns col)))
- nil)
-
-(defun find-or-make-col-in-columns (colname filename cols)
- (let ((col (find-col-in-columns colname filename cols)))
- (if col
- col
- ;; try to find column name without a terminal digit
- (let* ((last-char (char colname (1- (length colname))))
- (digit (- (char-code last-char) (char-code #\0))))
- (if (and (>= digit 0) (<= digit 9))
- (let ((base-colname (subseq colname 0 (1- (length colname)))))
- (setq col (find-col-in-columns base-colname filename cols))
- (if col
- (let ((new-col (make-umls-col
- :col (copy-seq colname)
- :des (copy-seq (umls-col-des col))
- :ref (copy-seq (umls-col-ref col))
- :min (umls-col-min col)
- :max (umls-col-max col)
- :fil (copy-seq (umls-col-fil col))
- :sqltype (copy-seq (umls-col-sqltype col))
- :dty (copy-seq (umls-col-dty col))
- :parsefunc (umls-col-parsefunc col)
- :quotechar (copy-seq (umls-col-quotechar col))
- :datatype (umls-col-datatype col)
- :custom-value-func (umls-col-custom-value-func col))))
- (push new-col *umls-cols*)
- new-col)
- (error "Couldn't find a base column for col ~A in file ~A"
- colname filename)))
- (let ((new-col (make-umls-col
- :col (copy-seq colname)
- :des "Unknown"
- :ref ""
- :min nil
- :max nil
- :fil filename
- :sqltype "VARCHAR"
- :dty nil
- :parsefunc #'add-sql-quotes
- :quotechar "'"
- :datatype nil
- :custom-value-func nil)))
- (push new-col *umls-cols*)
- new-col))))))
-
-(defun find-umls-col (colname filename)
+ (dolist (ucol ucols nil)
+ (when (and (string-equal filename (fil ucol))
+ (string-equal colname (col ucol)))
+ (return-from find-ucol-of-colname ucol))))
+
+(defun ensure-col-in-columns (colname filename ucols)
+ (aif (find-ucol-of-colname colname filename ucols)
+ it
+ (add-ucols (make-ucol-for-column colname filename ucols))))
+
+(defun make-ucol-for-column (colname filename ucols)
+ ;; try to find column name without a terminal digit
+ (let* ((len (length colname))
+ (last-digit? (digit-char-p (schar colname (1- len))))
+ (base-colname (if last-digit?
+ (subseq colname 0 (1- len))
+ colname))
+ (ucol (when last-digit?
+ (find-ucol-of-colname base-colname filename ucols))))
+ (when (and last-digit? (null ucol))
+ (error "Couldn't find a base column for col ~A in file ~A"
+ colname filename))
+ (copy-or-new-ucol colname filename ucol)))
+
+(defun copy-or-new-ucol (colname filename ucol)
+ (if ucol
+ (make-instance
+ 'ucol
+ :col (copy-seq colname) :des (copy-seq (des ucol)) :ref (copy-seq (ref ucol))
+ :min (cmin ucol) :max (cmax ucol) :fil (copy-seq (fil ucol))
+ :sqltype (copy-seq (sqltype ucol)) :dty (copy-seq (dty ucol))
+ :parse-fun (parse-fun ucol) :quote-str (copy-seq (quote-str ucol))
+ :datatype (datatype ucol) :custom-value-fun (custom-value-fun ucol))
+ (make-empty-ucol colname filename)))
+
+(defun ensure-compiled-fun (fun)
+ "Ensure that a function is compiled"
+ (etypecase fun
+ (null
+ nil)
+ (function
+ (if (compiled-function-p fun)
+ fun
+ (compile nil fun)))
+ (list
+ (compile nil fun))))
+
+(defun make-ucol (col des ref min av max fil dty
+ &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes)
+ (quote-str "'") (custom-value-fun))
+ (let ((ucol (make-instance
+ 'ucol
+ :col col :des des :ref ref :min min :av av
+ :max (if (eql max 0) 1 max) ;; ensure at least one char wide
+ :fil fil
+ :dty dty
+ :sqltype sqltype
+ :quote-str quote-str
+ :parse-fun (ensure-compiled-fun parse-fun)
+ :custom-value-fun (ensure-compiled-fun custom-value-fun))))
+ (ensure-ucol-datatype ucol (datatype-for-colname col))
+ ucol))
+
+(defun make-empty-ucol (colname filename)
+ ;;(format "call in make-empty-ucol: ~A/~A" colname filename)
+ (make-ucol (copy-seq colname) "Unknown" "" nil nil nil filename nil))
+
+(defun find-ucol (colname filename)
"Returns list of umls-col structure for a column name and a filename"
- (find-or-make-col-in-columns colname filename *umls-cols*))
-
-(defun find-umls-file (filename)
- "Returns umls-file structure for a filename"
- (find-if (lambda (f) (string-equal filename (umls-file-fil f))) *umls-files*))
-
-(defun umls-cols-for-umls-file (file)
- "Returns list of umls-cols for a file structure"
- (let ((filename (umls-file-fil file)))
- (mapcar (lambda (col) (find-umls-col col filename))
- (umls-file-fields file))))
-
-
-;;; Routines for analyzing cost of fixed size storage
-
-(defun umls-fixed-size-waste ()
- "Display storage waste if using all fixed size storage"
- (let ((totalwaste 0)
- (totalunavoidable 0)
- (totalavoidable 0)
- (unavoidable '())
- (avoidable '()))
- (dolist (file *umls-files*)
- (dolist (col (umls-file-colstructs file))
- (let* ((avwaste (- (umls-col-max col) (umls-col-av col)))
- (cwaste (* avwaste (umls-file-rws file))))
- (unless (zerop cwaste)
- (if (<= avwaste 6)
- (progn
- (incf totalunavoidable cwaste)
- (setq unavoidable (append unavoidable (list (list (umls-file-fil file) (umls-col-col col) avwaste cwaste)))))
- (progn
- (incf totalavoidable cwaste)
- (setq avoidable (append avoidable (list (list (umls-file-fil file) (umls-col-col col) avwaste cwaste))))))
- (incf totalwaste cwaste)))))
- (values totalwaste totalavoidable totalunavoidable avoidable unavoidable)))
-
-(defun display-waste ()
- (unless *umls-files*
- (init-umls))
- (multiple-value-bind (tw ta tu al ul) (umls-fixed-size-waste)
- (format t "Total waste: ~d~%" tw)
- (format t "Total avoidable: ~d~%" ta)
- (format t "Total unavoidable: ~d~%" tu)
- (format t "Avoidable:~%")
- (dolist (w al)
- (format t " (~a,~a): ~a,~a~%" (car w) (cadr w) (caddr w) (cadddr w)))
- (format t "Unavoidable:~%")
- (dolist (w ul)
- (format t " (~a,~a): ~a,~a~%" (car w) (cadr w) (caddr w) (cadddr w)))
- ))
-
-(defun max-umls-field ()
- "Return length of longest field"
- (unless *umls-files*
- (init-umls))
- (let ((max 0))
- (declare (fixnum max))
- (dolist (col *umls-cols*)
- (when (> (umls-col-max col) max)
- (setq max (umls-col-max col))))
- max))
-
-(defun max-umls-row ()
- "Return length of longest row"
- (if t
- 6000 ;;; hack to use on systems without MRCOLS/MRFILES -- ok for UMLS2001
- (progn
- (unless *umls-files*
- (init-umls))
- (let ((rowsizes '()))
- (dolist (file *umls-files*)
- (let ((row 0)
- (fields (umls-file-colstructs file)))
- (dolist (field fields)
- (incf row (1+ (umls-col-max field))))
- (push row rowsizes)))
- (car (sort rowsizes #'>))))))
+ (ensure-col-in-columns colname filename *umls-cols*))
+
+(defun find-ufile (filename)
+ "Returns umls-file structure for a filename"
+ (find-if #'(lambda (f) (string= filename (fil f))) *umls-files*))
+
+(defun position-field-file (filename fieldname)
+ "Returns the position of a field in a file"
+ (let ((ufile (find-ufile filename)))
+ (unless ufile
+ (warn "Unable to find ufile for filename ~A." filename)
+ (return-from position-field-file nil))
+ (let ((pos (position fieldname (fields ufile) :test #'string=)))
+ (unless pos
+ (warn "Unable to find field ~A in ufile ~S." fieldname ufile)
+ (return-from position-field-file nil))
+ pos)))
+
+(defun find-ucols-for-ufile (ufile)
+ "Returns list of umls-cols for a file structure"
+ (loop for colname in (fields ufile)
+ collect (find-ucol colname
+ (if (subdir ufile)
+ (concatenate 'string (subdir ufile) "/" (fil ufile))
+ (fil ufile)))))
+
+(defun umls-field-string-to-list (fmt)
+ "Converts a comma delimited list of fields into a list of field names. Will
+append a unique number (starting at 2) onto a column name that is repeated in the list"
+ (let ((col-counts (make-hash-table :test 'equal)))
+ (loop for colname in (delimited-string-to-list (escape-column-name fmt) #\,)
+ collect
+ (multiple-value-bind (value found) (gethash colname col-counts)
+ (cond
+ (found
+ (incf (gethash colname col-counts))
+ (concatenate 'string colname (write-to-string (1+ value))))
+ (t
+ (setf (gethash colname col-counts) 1)
+ colname))))))
+
+(defun decompose-fil (fil)
+ (if fil
+ (let ((pos (position #\/ fil)))
+ (if pos
+ (values (subseq fil (1+ pos)) (subseq fil 0 pos))
+ (values fil nil)))
+ (values nil nil)))
+
+(defun filename-to-tablename (file)
+ (let ((pos (search ".RRF" file)))
+ (when pos
+ (setf file (subseq file 0 pos))))
+ (substitute #\_ #\. file))
+
+(defun make-ufile (dir fil des cls rws bts fields)
+ (multiple-value-bind (file subdir) (decompose-fil fil)
+ (let ((ufile (make-instance 'ufile :dir dir :fil file :subdir subdir
+ :des des :cls cls
+ :rws rws :bts bts :fields fields
+ :table (filename-to-tablename file))))
+ ufile)))
+
+(defun set-ucols-for-ufiles (ufiles)
+ (dolist (ufile ufiles)
+ (setf (ucols ufile) (find-ucols-for-ufile ufile))))
+
+(defun datatype-for-colname (colname)
+"Return datatype for column name"
+ (second (find colname +col-datatypes+ :key #'car :test #'string-equal)))
+
+(defun canonicalize-column-type (type)
+ (cond
+ ((string-equal type "TINYINT")
+ (case *umls-sql-type*
+ (:mysql "TINYINT")
+ ((:postgresql :postgresql-socket) "INT1")
+ (:oracle "NUMBER(3,0)")
+ (t "INTEGER")))
+ ((string-equal type "SMALLINT")
+ (case *umls-sql-type*
+ (:mysql "SMALLINT")
+ ((:postgresql :postgresql-socket) "INT2")
+ (:oracle "NUMBER(5,0)")
+ (t "INTEGER")))
+ ((string-equal type "INTEGER")
+ (case *umls-sql-type*
+ (:mysql "INTEGER")
+ ((:postgresql :postgresql-socket) "INT4")
+ (:oracle "NUMBER(9,0)")
+ (t "INTEGER")))
+ ((string-equal type "BIGINT")
+ (case *umls-sql-type*
+ (:mysql "BIGINT")
+ ((:postgresql :postgresql-socket) "INT8")
+ (:oracle "NUMBER(38,0)")
+ (t "INTEGER")))
+ ((string-equal type "TEXT")
+ (case *umls-sql-type*
+ (:mysql "TEXT")
+ ((:postgresql :postgresql-socket) "TEXT")
+ (:oracle "VARCHAR2(3000)")
+ (t "VARCHAR(3000)")))
+ ((string-equal type "VARCHAR")
+ (case *umls-sql-type*
+ (:mysql "VARCHAR")
+ ((:postgresql :postgresql-socket) "VARCHAR")
+ (:oracle "VARCHAR2")
+ (t "VARCHAR")))
+ ((string-equal type "NUMERIC")
+ (case *umls-sql-type*
+ (:mysql "NUMERIC")
+ ((:postgresql :postgresql-socket) "NUMERIC")
+ (:oracle "NUMBER")
+ (t "NUMERIC")))
+ (t
+ type)))
+
+(defun ensure-ucol-datatype (col datatype)
+ "Add data type information to column"
+ (setf (datatype col) datatype)
+ (case datatype
+ (sql-u (setf (sqltype col) (canonicalize-column-type "INTEGER")
+ (parse-fun col) #'parse-ui
+ (quote-str col) ""))
+ (sql-s (setf (sqltype col) (canonicalize-column-type "SMALLINT")
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
+ (sql-l (setf (sqltype col) (canonicalize-column-type "BIGINT")
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
+ (sql-i (setf (sqltype col) (canonicalize-column-type "INTEGER")
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
+ (sql-t (setf (sqltype col) (canonicalize-column-type "TINYINT")
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
+ (sql-f (setf (sqltype col) (canonicalize-column-type "NUMERIC")
+ (parse-fun col) #'read-from-string
+ (quote-str col) ""))
+ (t ; Default column type, optimized text storage
+ (setf (parse-fun col) #'add-sql-quotes
+ (quote-str col) "'")
+ (when (and (cmax col) (av col))
+ (if (> (cmax col) 255)
+ (setf (sqltype col) (canonicalize-column-type "TEXT"))
+ (setf (sqltype col) (canonicalize-column-type "VARCHAR")))))))
+
+(defun escape-column-name (name)
+ (substitute #\_ #\/ name))
+
+;; SQLNAME is required for collision of SQL reserved words (MYSQL 8: RANK)
+;; and column names in UMLS (RANK in MRRANK)
+(defvar *sql-reserved-names* '("RANK"))
+(defmethod sqlname ((c ucol))
+ (sqlname (col c)))
+(defmethod sqlname ((name string))
+ (if (find name *sql-reserved-names* :test #'string-equal)
+ (concatenate 'string "_" name)
+ name))
+(defmethod sqlname ((l list))
+ (mapcar #'sqlname l))
+(defmethod sqlname ((s symbol))
+ (sqlname (symbol-name s)))
projects / umlisp.git / blobdiff