(defun ensure-ucols+ufiles (&optional (alwaysclear nil))
"Initialize all UMLS file and column structures if not already initialized"
- (when (or alwaysclear (null *umls-files*))
- (setq *umls-cols* nil)
- (setq *umls-files* nil)
- (gen-ucols)
- (gen-ufiles)
- (ensure-field-lengths)))
+ (handler-case
+ (when (or alwaysclear (null *umls-files*))
+ (setf *umls-cols* nil *umls-files* nil)
+ (gen-ufiles)
+ (gen-ucols)
+ (set-ucols-for-ufiles *umls-files*)
+ (ensure-field-lengths))
+ (error (e)
+ (warn "Error reading ucols+ufiles: ~A." e)
+ (setf *umls-cols* nil *umls-files* nil)
+ nil))
+ t)
+
(defun add-ucols (ucols)
"Adds a ucol or list of ucols to *umls-cols*. Returns input value."
(defun ufile-pathname (ufile &optional (extension ""))
"Return pathname for a umls filename with an optional extension"
(assert (typep ufile 'ufile))
- (let ((dirs (append (list (dir ufile))
- (awhen (subdir ufile) (list it)))))
- (merge-pathnames
- (make-pathname :name (concatenate 'string (fil ufile) extension)
+ (let* ((dirs (append (list (dir ufile))
+ (awhen (subdir ufile) (list it))))
+ (name-list (delimited-string-to-list (fil ufile) #\.))
+ (name (if (second name-list)
+ (first name-list)
+ (concatenate 'string (first name-list) (or extension ""))))
+ (type (when (second name-list)
+ (concatenate 'string (second name-list) (or extension "")))))
+ (merge-pathnames
+ (make-pathname :name name :type type
:directory (cons :relative dirs))
*umls-path*)))
"Return pathname for a umls filename with an optional extension"
(etypecase filename
(string
- (merge-pathnames
- (make-pathname :name (concatenate 'string filename extension))
+ (let* ((name-list (delimited-string-to-list filename #\.))
+ (name (if (second name-list)
+ (first name-list)
+ (concatenate 'string (first name-list) (or extension ""))))
+ (type (when (second name-list)
+ (concatenate 'string (second name-list) (or extension "")))))
+ (merge-pathnames
+ (make-pathname :name name :type type)
(case (schar filename 0)
- ((#\M #\m)
- *meta-path*)
- ((#\L #\l)
- *lex-path*)
- ((#\S #\s)
- *net-path*)
- (t
- *umls-path*))))
+ ((#\M #\m)
+ *meta-path*)
+ ((#\L #\l)
+ *lex-path*)
+ ((#\S #\s)
+ *net-path*)
+ (t
+ *umls-path*)))))
(pathname
filename)))
(error "Can't find ~A filename in ufiles" filename))
(unless (= (length fields-max) (length (fields file)))
(error
- "Number of file fields ~A not equal to field count in ufile ~S"
+ "Number of file fields ~A not equal to field count in ufile ~S"
fields-max file))
(dotimes (i (length (fields file)))
(declare (fixnum i))
(setf (cmax col) (aref fields-max i))
(setf (av col) (aref fields-av i))
(ensure-ucol-datatype col (datatype-for-colname (col col)))))))))
-
+
(defun ufiles-to-measure ()
"Returns a list of ufiles to measure"
(loop for ufile in *umls-files*
unless (or (char= #\M (schar (fil ufile) 0))
(char= #\m (schar (fil ufile) 0)))
collect ufile))
-
-
+
+
(defun ufiles-field-lengths (ufiles)
"Returns a list of lists of containing (FILE MAX AV)"
(loop for ufile in ufiles collect (file-field-lengths ufile)))
"Returns a list of FILENAME MAX AV"
(declare (optimize (speed 3) (safety 0)))
(let (fields-max fields-av num-fields (count-lines 0))
+ (declare (fixnum count-lines))
(with-umls-ufile (line ufile)
(unless num-fields
(setq num-fields (length line))
- (setq fields-max (make-array num-fields :element-type 'fixnum
+ (setq fields-max (make-array num-fields :element-type 'fixnum
:initial-element 0))
- (setq fields-av (make-array num-fields :element-type 'number
+ (setq fields-av (make-array num-fields :element-type '(or integer float)
:initial-element 0)))
(dotimes (i num-fields)
(declare (fixnum i))
- (let ((len (length (nth i line))))
+ (let* ((str (nth i line))
+ (len (length #-(and clisp unicode) str
+ #+(and clisp unicode)
+ (if *octet-sql-storage*
+ (ext:convert-string-to-bytes str charset:utf-8)
+ str))))
+ #-(and clisp unicode) (declare (string str))
+ (declare (type (integer 0 10000000) len))
(incf (aref fields-av i) len)
(when (> len (aref fields-max i))
(setf (aref fields-max i) len))))
(quote-str "'") (custom-value-fun))
(let ((ucol (make-instance
'ucol
- :col col :des des :ref ref :min min :av av
+ :col col :des des :ref ref :min min :av av
:max (if (eql max 0) 1 max) ;; ensure at least one char wide
:fil fil
:dty dty
- :sqltype sqltype
+ :sqltype sqltype
:quote-str quote-str
:parse-fun (ensure-compiled-fun parse-fun)
:custom-value-fun (ensure-compiled-fun custom-value-fun))))
(ensure-col-in-columns colname filename *umls-cols*))
(defun find-ufile (filename)
- "Returns umls-file structure for a filename"
- (find-if #'(lambda (f) (string-equal filename (fil f))) *umls-files*))
+ "Returns umls-file structure for a filename"
+ (find-if #'(lambda (f) (string= filename (fil f))) *umls-files*))
+
+(defvar *position-hash* (make-hash-table :test 'equal))
+
+(defun position-field-file (filename fieldname)
+ "Returns the position of a field in a file"
+ (let ((key (cons filename fieldname)))
+ (multiple-value-bind (pos found) (gethash key *position-hash*)
+ (if found
+ (return-from position-field-file pos)))
+
+ (let ((ufile (find-ufile filename)))
+ (unless ufile
+ (warn "Unable to find ufile for filename ~A." filename)
+ (return-from position-field-file nil))
+ (let ((pos (position fieldname (fields ufile) :test #'string=)))
+ (unless pos
+ (warn "Unable to find field ~A in ufile ~S." fieldname ufile)
+ (return-from position-field-file nil))
+ (setf (gethash key *position-hash*) pos)
+ pos))))
(defun find-ucols-for-ufile (ufile)
"Returns list of umls-cols for a file structure"
(loop for colname in (fields ufile)
- collect (find-ucol colname
+ collect (find-ucol colname
(if (subdir ufile)
(concatenate 'string (subdir ufile) "/" (fil ufile))
(fil ufile)))))
:des des :cls cls
:rws rws :bts bts :fields fields
:table (filename-to-tablename file))))
- (setf (ucols ufile) (find-ucols-for-ufile ufile))
ufile)))
+(defun set-ucols-for-ufiles (ufiles)
+ (dolist (ufile ufiles)
+ (setf (ucols ufile) (find-ucols-for-ufile ufile))))
+
(defun datatype-for-colname (colname)
-"Return datatype for column name"
+"Return datatype for column name"
(second (find colname +col-datatypes+ :key #'car :test #'string-equal)))
(defun canonicalize-column-type (type)
(cond
+ ((string-equal type "TINYINT")
+ (case *umls-sql-type*
+ (:mysql "TINYINT")
+ ((:postgresql :postgresql-socket) "INT1")
+ (:oracle "NUMBER(3,0)")
+ (t "INTEGER")))
((string-equal type "SMALLINT")
(case *umls-sql-type*
(:mysql "SMALLINT")
(t "NUMERIC")))
(t
type)))
-
+
(defun ensure-ucol-datatype (col datatype)
"Add data type information to column"
(setf (datatype col) datatype)
(sql-i (setf (sqltype col) (canonicalize-column-type "INTEGER")
(parse-fun col) #'parse-integer
(quote-str col) ""))
+ (sql-t (setf (sqltype col) (canonicalize-column-type "TINYINT")
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
(sql-f (setf (sqltype col) (canonicalize-column-type "NUMERIC")
(parse-fun col) #'read-from-string
(quote-str col) ""))
(t ; Default column type, optimized text storage
- (setf (parse-fun col) #'add-sql-quotes
+ (setf (parse-fun col) #'add-sql-quotes
(quote-str col) "'")
(when (and (cmax col) (av col))
(if (> (cmax col) 255)
projects / umlisp.git / blobdiff