;;;; Author: Kevin M. Rosenberg
;;;; Created: Apr 2000
;;;;
-;;;; $Id$
-;;;;
;;;; This file, part of UMLisp, is
-;;;; Copyright (c) 2000-2006 by Kevin M. Rosenberg, M.D.
+;;;; Copyright (c) 2000-2010 by Kevin M. Rosenberg, M.D.
;;;;
;;;; UMLisp users are granted the rights to distribute and use this software
;;;; as governed by the terms of the GNU General Public License.
(handler-case
(when (or alwaysclear (null *umls-files*))
(setf *umls-cols* nil *umls-files* nil)
- (gen-ucols)
(gen-ufiles)
+ (gen-ucols)
+ (set-ucols-for-ufiles *umls-files*)
(ensure-field-lengths))
(error (e)
+ (warn "Error reading ucols+ufiles: ~A." e)
(setf *umls-cols* nil *umls-files* nil)
- (error e)))
+ nil))
t)
(defun ufile-pathname (ufile &optional (extension ""))
"Return pathname for a umls filename with an optional extension"
(assert (typep ufile 'ufile))
- (let ((dirs (append (list (dir ufile))
- (awhen (subdir ufile) (list it)))))
+ (let* ((dirs (nconc (list (dir ufile))
+ (awhen (subdir ufile) (list it))))
+ (name-list (delimited-string-to-list (fil ufile) #\.))
+ (name (if (second name-list)
+ (first name-list)
+ (concatenate 'string (first name-list) (or extension ""))))
+ (type (when (second name-list)
+ (concatenate 'string (second name-list) (or extension "")))))
(merge-pathnames
- (make-pathname :name (concatenate 'string (fil ufile) extension)
- :directory (cons :relative dirs))
+ (make-pathname :name name :type type
+ :directory (cons :relative dirs))
*umls-path*)))
(defun umls-pathname (filename &optional (extension ""))
"Return pathname for a umls filename with an optional extension"
(etypecase filename
(string
+ (let* ((name-list (delimited-string-to-list filename #\.))
+ (name (if (second name-list)
+ (first name-list)
+ (concatenate 'string (first name-list) (or extension ""))))
+ (type (when (second name-list)
+ (concatenate 'string (second name-list) (or extension "")))))
(merge-pathnames
- (make-pathname :name (concatenate 'string filename extension))
+ (make-pathname :name name :type type)
(case (schar filename 0)
- ((#\M #\m)
- *meta-path*)
- ((#\L #\l)
- *lex-path*)
- ((#\S #\s)
- *net-path*)
- (t
- *umls-path*))))
+ ((#\M #\m)
+ *meta-path*)
+ ((#\L #\l)
+ *lex-path*)
+ ((#\S #\s)
+ *net-path*)
+ (t
+ *umls-path*)))))
(pathname
filename)))
"Initial colstruct field lengths for files that don't have a measurement.
Currently, these are the LEX and NET files."
(dolist (length-list (ufiles-field-lengths (ufiles-to-measure)))
- (destructuring-bind (filename fields-max fields-av) length-list
+ (destructuring-bind (filename fields-max fields-av count-lines) length-list
(let ((file (find-ufile filename)))
- (unless file
- (error "Can't find ~A filename in ufiles" filename))
- (unless (= (length fields-max) (length (fields file)))
- (error
- "Number of file fields ~A not equal to field count in ufile ~S"
- fields-max file))
- (dotimes (i (length (fields file)))
- (declare (fixnum i))
- (let* ((field (nth i (fields file)))
- (col (find-ucol field filename)))
- (unless col
- (error "can't find column ~A" field))
- (setf (cmax col) (aref fields-max i))
- (setf (av col) (aref fields-av i))
- (ensure-ucol-datatype col (datatype-for-colname (col col)))))))))
+ (unless file
+ (error "Can't find ~A filename in ufiles" filename))
+ (if (zerop count-lines)
+ (warn "File ~A is empty." filename)
+ (progn
+ (unless (= (length fields-max) (length (fields file)))
+ (error
+ "Number of file fields ~A not equal to field count in ufile ~S"
+ fields-max file))
+ (dotimes (i (length (fields file)))
+ (declare (fixnum i))
+ (let* ((field (nth i (fields file)))
+ (col (find-ucol field filename)))
+ (unless col
+ (error "can't find column ~A" field))
+ (setf (cmax col) (aref fields-max i))
+ (setf (av col) (aref fields-av i))
+ (ensure-ucol-datatype col (datatype-for-colname (col col)))))))))))
(defun ufiles-to-measure ()
"Returns a list of ufiles to measure"
(loop for ufile in *umls-files*
- unless (or (char= #\M (schar (fil ufile) 0))
- (char= #\m (schar (fil ufile) 0)))
+ unless (or (char= #\M (schar (fil ufile) 0))
+ (char= #\m (schar (fil ufile) 0)))
collect ufile))
"Returns a list of FILENAME MAX AV"
(declare (optimize (speed 3) (safety 0)))
(let (fields-max fields-av num-fields (count-lines 0))
+ (declare (fixnum count-lines))
(with-umls-ufile (line ufile)
(unless num-fields
- (setq num-fields (length line))
- (setq fields-max (make-array num-fields :element-type 'fixnum
- :initial-element 0))
- (setq fields-av (make-array num-fields :element-type 'number
- :initial-element 0)))
- (dotimes (i num-fields)
- (declare (fixnum i))
- (let* ((str (nth i line))
+ (setq num-fields (length line))
+ (setq fields-max (make-array num-fields :element-type 'fixnum
+ :initial-element 0))
+ (setq fields-av (make-array num-fields :element-type '(or integer float)
+ :initial-element 0)))
+ (dotimes (i (or num-fields 0))
+ (declare (fixnum i))
+ (let* ((str (nth i line))
(len (length #-(and clisp unicode) str
#+(and clisp unicode)
(if *octet-sql-storage*
(ext:convert-string-to-bytes str charset:utf-8)
str))))
- (incf (aref fields-av i) len)
- (when (> len (aref fields-max i))
- (setf (aref fields-max i) len))))
+ #-(and clisp unicode) (declare (string str))
+ (declare (type (integer 0 100000000) len))
+ (incf (aref fields-av i) len)
+ (when (> len (aref fields-max i))
+ (setf (aref fields-max i) len))))
(incf count-lines))
- (dotimes (i num-fields)
- (setf (aref fields-av i) (float (/ (aref fields-av i) count-lines))))
- (list (fil ufile) fields-max fields-av)))
+ (dotimes (i (or num-fields 0))
+ (setf (aref fields-av i)
+ (if (plusp count-lines)
+ (float (/ (aref fields-av i) count-lines))
+ 0)))
+ (list (fil ufile) fields-max fields-av count-lines)))
;;; UMLS column/file functions
"Returns list of umls-col structure for a column name and a filename"
(dolist (ucol ucols nil)
(when (and (string-equal filename (fil ucol))
- (string-equal colname (col ucol)))
+ (string-equal colname (col ucol)))
(return-from find-ucol-of-colname ucol))))
(defun ensure-col-in-columns (colname filename ucols)
(defun make-ucol-for-column (colname filename ucols)
;; try to find column name without a terminal digit
(let* ((len (length colname))
- (last-digit? (digit-char-p (schar colname (1- len))))
- (base-colname (if last-digit?
- (subseq colname 0 (1- len))
- colname))
- (ucol (when last-digit?
- (find-ucol-of-colname base-colname filename ucols))))
+ (last-digit? (digit-char-p (schar colname (1- len))))
+ (base-colname (if last-digit?
+ (subseq colname 0 (1- len))
+ colname))
+ (ucol (when last-digit?
+ (find-ucol-of-colname base-colname filename ucols))))
(when (and last-digit? (null ucol))
(error "Couldn't find a base column for col ~A in file ~A"
- colname filename))
+ colname filename))
(copy-or-new-ucol colname filename ucol)))
(defun copy-or-new-ucol (colname filename ucol)
nil)
(function
(if (compiled-function-p fun)
- fun
- (compile nil fun)))
+ fun
+ (compile nil fun)))
(list
(compile nil fun))))
(defun make-ucol (col des ref min av max fil dty
- &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes)
- (quote-str "'") (custom-value-fun))
+ &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes)
+ (quote-str "'") (custom-value-fun))
(let ((ucol (make-instance
- 'ucol
- :col col :des des :ref ref :min min :av av
- :max (if (eql max 0) 1 max) ;; ensure at least one char wide
- :fil fil
- :dty dty
- :sqltype sqltype
- :quote-str quote-str
- :parse-fun (ensure-compiled-fun parse-fun)
- :custom-value-fun (ensure-compiled-fun custom-value-fun))))
+ 'ucol
+ :col col :des des :ref ref :min min :av av
+ :max (if (eql max 0) 1 max) ;; ensure at least one char wide
+ :fil fil
+ :dty dty
+ :sqltype sqltype
+ :quote-str quote-str
+ :parse-fun (ensure-compiled-fun parse-fun)
+ :custom-value-fun (ensure-compiled-fun custom-value-fun))))
(ensure-ucol-datatype ucol (datatype-for-colname col))
ucol))
(defun find-ufile (filename)
"Returns umls-file structure for a filename"
- (find-if #'(lambda (f) (string-equal filename (fil f))) *umls-files*))
+ (find-if #'(lambda (f) (string= filename (fil f))) *umls-files*))
+
+(defun position-field-file (filename fieldname)
+ "Returns the position of a field in a file"
+ (let ((ufile (find-ufile filename)))
+ (unless ufile
+ (warn "Unable to find ufile for filename ~A." filename)
+ (return-from position-field-file nil))
+ (let ((pos (position fieldname (fields ufile) :test #'string=)))
+ (unless pos
+ (warn "Unable to find field ~A in ufile ~S." fieldname ufile)
+ (return-from position-field-file nil))
+ pos)))
(defun find-ucols-for-ufile (ufile)
"Returns list of umls-cols for a file structure"
(loop for colname in (fields ufile)
collect (find-ucol colname
- (if (subdir ufile)
- (concatenate 'string (subdir ufile) "/" (fil ufile))
- (fil ufile)))))
+ (if (subdir ufile)
+ (concatenate 'string (subdir ufile) "/" (fil ufile))
+ (fil ufile)))))
(defun umls-field-string-to-list (fmt)
"Converts a comma delimited list of fields into a list of field names. Will
append a unique number (starting at 2) onto a column name that is repeated in the list"
(let ((col-counts (make-hash-table :test 'equal)))
(loop for colname in (delimited-string-to-list (escape-column-name fmt) #\,)
- collect
- (multiple-value-bind (value found) (gethash colname col-counts)
- (cond
- (found
- (incf (gethash colname col-counts))
- (concatenate 'string colname (write-to-string (1+ value))))
- (t
- (setf (gethash colname col-counts) 1)
- colname))))))
+ collect
+ (multiple-value-bind (value found) (gethash colname col-counts)
+ (cond
+ (found
+ (incf (gethash colname col-counts))
+ (concatenate 'string colname (write-to-string (1+ value))))
+ (t
+ (setf (gethash colname col-counts) 1)
+ colname))))))
(defun decompose-fil (fil)
(if fil
(let ((pos (position #\/ fil)))
- (if pos
- (values (subseq fil (1+ pos)) (subseq fil 0 pos))
- (values fil nil)))
+ (if pos
+ (values (subseq fil (1+ pos)) (subseq fil 0 pos))
+ (values fil nil)))
(values nil nil)))
(defun filename-to-tablename (file)
(defun make-ufile (dir fil des cls rws bts fields)
(multiple-value-bind (file subdir) (decompose-fil fil)
(let ((ufile (make-instance 'ufile :dir dir :fil file :subdir subdir
- :des des :cls cls
- :rws rws :bts bts :fields fields
- :table (filename-to-tablename file))))
- (setf (ucols ufile) (find-ucols-for-ufile ufile))
+ :des des :cls cls
+ :rws rws :bts bts :fields fields
+ :table (filename-to-tablename file))))
ufile)))
+(defun set-ucols-for-ufiles (ufiles)
+ (dolist (ufile ufiles)
+ (setf (ucols ufile) (find-ucols-for-ufile ufile))))
+
(defun datatype-for-colname (colname)
"Return datatype for column name"
(second (find colname +col-datatypes+ :key #'car :test #'string-equal)))
(defun canonicalize-column-type (type)
(cond
+ ((string-equal type "TINYINT")
+ (case *umls-sql-type*
+ (:mysql "TINYINT")
+ ((:postgresql :postgresql-socket) "INT1")
+ (:oracle "NUMBER(3,0)")
+ (t "INTEGER")))
((string-equal type "SMALLINT")
(case *umls-sql-type*
(:mysql "SMALLINT")
(setf (datatype col) datatype)
(case datatype
(sql-u (setf (sqltype col) (canonicalize-column-type "INTEGER")
- (parse-fun col) #'parse-ui
- (quote-str col) ""))
+ (parse-fun col) #'parse-ui
+ (quote-str col) ""))
(sql-s (setf (sqltype col) (canonicalize-column-type "SMALLINT")
- (parse-fun col) #'parse-integer
- (quote-str col) ""))
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
(sql-l (setf (sqltype col) (canonicalize-column-type "BIGINT")
- (parse-fun col) #'parse-integer
- (quote-str col) ""))
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
(sql-i (setf (sqltype col) (canonicalize-column-type "INTEGER")
- (parse-fun col) #'parse-integer
- (quote-str col) ""))
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
+ (sql-t (setf (sqltype col) (canonicalize-column-type "TINYINT")
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
(sql-f (setf (sqltype col) (canonicalize-column-type "NUMERIC")
- (parse-fun col) #'read-from-string
- (quote-str col) ""))
- (t ; Default column type, optimized text storage
+ (parse-fun col) #'read-from-string
+ (quote-str col) ""))
+ (t ; Default column type, optimized text storage
(setf (parse-fun col) #'add-sql-quotes
- (quote-str col) "'")
+ (quote-str col) "'")
(when (and (cmax col) (av col))
(if (> (cmax col) 255)
- (setf (sqltype col) (canonicalize-column-type "TEXT"))
- (setf (sqltype col) (canonicalize-column-type "VARCHAR")))))))
+ (setf (sqltype col) (canonicalize-column-type "TEXT"))
+ (setf (sqltype col) (canonicalize-column-type "VARCHAR")))))))
(defun escape-column-name (name)
(substitute #\_ #\/ name))
+
+;; SQLNAME is required for collision of SQL reserved words (MYSQL 8: RANK)
+;; and column names in UMLS (RANK in MRRANK)
+(defvar *sql-reserved-names* '("RANK"))
+(defmethod sqlname ((c ucol))
+ (sqlname (col c)))
+(defmethod sqlname ((name string))
+ (if (find name *sql-reserved-names* :test #'string-equal)
+ (concatenate 'string "_" name)
+ name))
+(defmethod sqlname ((l list))
+ (mapcar #'sqlname l))
+(defmethod sqlname ((s symbol))
+ (sqlname (symbol-name s)))
projects / umlisp.git / blobdiff