X-Git-Url: http://git.kpe.io/?p=umlisp.git;a=blobdiff_plain;f=parse-common.lisp;h=c9adcf740cd7d856a383723af8a99f55922cbf42;hp=ebd88a895beaaf9429777c10119d5314de4b5ab4;hb=7aadf2bbca30301d2479a21edbcdcb6e12da5cd0;hpb=b9fe7fe8b8e24133538f78dbaf6af73b5f0bdec2 diff --git a/parse-common.lisp b/parse-common.lisp index ebd88a8..c9adcf7 100644 --- a/parse-common.lisp +++ b/parse-common.lisp @@ -20,12 +20,17 @@ (defun ensure-ucols+ufiles (&optional (alwaysclear nil)) "Initialize all UMLS file and column structures if not already initialized" - (when (or alwaysclear (null *umls-files*)) - (setq *umls-cols* nil) - (setq *umls-files* nil) - (gen-ucols) - (gen-ufiles) - (ensure-field-lengths))) + (handler-case + (when (or alwaysclear (null *umls-files*)) + (setf *umls-cols* nil *umls-files* nil) + (gen-ucols) + (gen-ufiles) + (ensure-field-lengths)) + (error (e) + (setf *umls-cols* nil *umls-files* nil) + (error e))) + t) + (defun add-ucols (ucols) "Adds a ucol or list of ucols to *umls-cols*. Returns input value." @@ -40,10 +45,16 @@ (defun ufile-pathname (ufile &optional (extension "")) "Return pathname for a umls filename with an optional extension" (assert (typep ufile 'ufile)) - (let ((dirs (append (list (dir ufile)) - (awhen (subdir ufile) (list it))))) - (merge-pathnames - (make-pathname :name (concatenate 'string (fil ufile) extension) + (let* ((dirs (append (list (dir ufile)) + (awhen (subdir ufile) (list it)))) + (name-list (delimited-string-to-list (fil ufile) #\.)) + (name (if (second name-list) + (first name-list) + (concatenate 'string (first name-list) (or extension "")))) + (type (when (second name-list) + (concatenate 'string (second name-list) (or extension ""))))) + (merge-pathnames + (make-pathname :name name :type type :directory (cons :relative dirs)) *umls-path*))) @@ -51,17 +62,23 @@ "Return pathname for a umls filename with an optional extension" (etypecase filename (string - (merge-pathnames - (make-pathname :name (concatenate 'string filename extension)) + (let* ((name-list (delimited-string-to-list filename #\.)) + (name (if (second name-list) + (first name-list) + (concatenate 'string (first name-list) (or extension "")))) + (type (when (second name-list) + (concatenate 'string (second name-list) (or extension ""))))) + (merge-pathnames + (make-pathname :name name :type type) (case (schar filename 0) - ((#\M #\m) - *meta-path*) - ((#\L #\l) - *lex-path*) - ((#\S #\s) - *net-path*) - (t - *umls-path*)))) + ((#\M #\m) + *meta-path*) + ((#\L #\l) + *lex-path*) + ((#\S #\s) + *net-path*) + (t + *umls-path*))))) (pathname filename))) @@ -78,7 +95,7 @@ Currently, these are the LEX and NET files." (error "Can't find ~A filename in ufiles" filename)) (unless (= (length fields-max) (length (fields file))) (error - "Number of file fields ~A not equal to field count in ufile ~S" + "Number of file fields ~A not equal to field count in ufile ~S" fields-max file)) (dotimes (i (length (fields file))) (declare (fixnum i)) @@ -89,15 +106,15 @@ Currently, these are the LEX and NET files." (setf (cmax col) (aref fields-max i)) (setf (av col) (aref fields-av i)) (ensure-ucol-datatype col (datatype-for-colname (col col))))))))) - + (defun ufiles-to-measure () "Returns a list of ufiles to measure" (loop for ufile in *umls-files* unless (or (char= #\M (schar (fil ufile) 0)) (char= #\m (schar (fil ufile) 0))) collect ufile)) - - + + (defun ufiles-field-lengths (ufiles) "Returns a list of lists of containing (FILE MAX AV)" (loop for ufile in ufiles collect (file-field-lengths ufile))) @@ -109,13 +126,18 @@ Currently, these are the LEX and NET files." (with-umls-ufile (line ufile) (unless num-fields (setq num-fields (length line)) - (setq fields-max (make-array num-fields :element-type 'fixnum + (setq fields-max (make-array num-fields :element-type 'fixnum :initial-element 0)) (setq fields-av (make-array num-fields :element-type 'number :initial-element 0))) (dotimes (i num-fields) (declare (fixnum i)) - (let ((len (length (nth i line)))) + (let* ((str (nth i line)) + (len (length #-(and clisp unicode) str + #+(and clisp unicode) + (if *octet-sql-storage* + (ext:convert-string-to-bytes str charset:utf-8) + str)))) (incf (aref fields-av i) len) (when (> len (aref fields-max i)) (setf (aref fields-max i) len)))) @@ -180,11 +202,11 @@ Currently, these are the LEX and NET files." (quote-str "'") (custom-value-fun)) (let ((ucol (make-instance 'ucol - :col col :des des :ref ref :min min :av av + :col col :des des :ref ref :min min :av av :max (if (eql max 0) 1 max) ;; ensure at least one char wide :fil fil :dty dty - :sqltype sqltype + :sqltype sqltype :quote-str quote-str :parse-fun (ensure-compiled-fun parse-fun) :custom-value-fun (ensure-compiled-fun custom-value-fun)))) @@ -200,13 +222,13 @@ Currently, these are the LEX and NET files." (ensure-col-in-columns colname filename *umls-cols*)) (defun find-ufile (filename) - "Returns umls-file structure for a filename" + "Returns umls-file structure for a filename" (find-if #'(lambda (f) (string-equal filename (fil f))) *umls-files*)) (defun find-ucols-for-ufile (ufile) "Returns list of umls-cols for a file structure" (loop for colname in (fields ufile) - collect (find-ucol colname + collect (find-ucol colname (if (subdir ufile) (concatenate 'string (subdir ufile) "/" (fil ufile)) (fil ufile))))) @@ -250,7 +272,7 @@ append a unique number (starting at 2) onto a column name that is repeated in th ufile))) (defun datatype-for-colname (colname) -"Return datatype for column name" +"Return datatype for column name" (second (find colname +col-datatypes+ :key #'car :test #'string-equal))) (defun canonicalize-column-type (type) @@ -293,7 +315,7 @@ append a unique number (starting at 2) onto a column name that is repeated in th (t "NUMERIC"))) (t type))) - + (defun ensure-ucol-datatype (col datatype) "Add data type information to column" (setf (datatype col) datatype) @@ -314,7 +336,7 @@ append a unique number (starting at 2) onto a column name that is repeated in th (parse-fun col) #'read-from-string (quote-str col) "")) (t ; Default column type, optimized text storage - (setf (parse-fun col) #'add-sql-quotes + (setf (parse-fun col) #'add-sql-quotes (quote-str col) "'") (when (and (cmax col) (av col)) (if (> (cmax col) 255)