X-Git-Url: http://git.kpe.io/?p=umlisp.git;a=blobdiff_plain;f=parse-common.lisp;h=25d3ee3f11bc0dff11d8d1083dcb5a8bba3911ae;hp=06ad178466e3a1edbe54cacfbc6081c3f3039d9e;hb=57c4b059639968903aec88e65103a7263cb97535;hpb=2038ce2f480179fd23cb3ded6fa1250e61e22029 diff --git a/parse-common.lisp b/parse-common.lisp index 06ad178..25d3ee3 100644 --- a/parse-common.lisp +++ b/parse-common.lisp @@ -2,15 +2,15 @@ ;;;; ************************************************************************* ;;;; FILE IDENTIFICATION ;;;; -;;;; Name: parse-common.lisp -;;;; Purpose: Common, stable parsing routines for UMLisp -;;;; Author: Kevin M. Rosenberg -;;;; Date Started: Apr 2000 +;;;; Name: parse-common.lisp +;;;; Purpose: Common, stable parsing routines for UMLisp +;;;; Author: Kevin M. Rosenberg +;;;; Created: Apr 2000 ;;;; -;;;; $Id: parse-common.lisp,v 1.15 2003/06/10 22:30:16 kevin Exp $ +;;;; $Id$ ;;;; ;;;; This file, part of UMLisp, is -;;;; Copyright (c) 2000-2003 by Kevin M. Rosenberg, M.D. +;;;; Copyright (c) 2000-2006 by Kevin M. Rosenberg, M.D. ;;;; ;;;; UMLisp users are granted the rights to distribute and use this software ;;;; as governed by the terms of the GNU General Public License. @@ -18,15 +18,19 @@ (in-package #:umlisp) -(eval-when (:compile-toplevel) - (declaim (optimize (speed 3) (safety 1) (compilation-speed 0) (debug 3)))) - (defun ensure-ucols+ufiles (&optional (alwaysclear nil)) -"Initialize all UMLS file and column structures if not already initialized" - (when (or alwaysclear (null *umls-files*)) - (gen-ucols) - (gen-ufiles) - (ensure-field-lengths))) + "Initialize all UMLS file and column structures if not already initialized" + (handler-case + (when (or alwaysclear (null *umls-files*)) + (setf *umls-cols* nil *umls-files* nil) + (gen-ucols) + (gen-ufiles) + (ensure-field-lengths)) + (error (e) + (setf *umls-cols* nil *umls-files* nil) + (error e))) + t) + (defun add-ucols (ucols) "Adds a ucol or list of ucols to *umls-cols*. Returns input value." @@ -38,13 +42,29 @@ (setq *umls-files* (append (mklist ufiles) *umls-files*)) ufiles) +(defun ufile-pathname (ufile &optional (extension "")) + "Return pathname for a umls filename with an optional extension" + (assert (typep ufile 'ufile)) + (let* ((dirs (append (list (dir ufile)) + (awhen (subdir ufile) (list it)))) + (name-list (delimited-string-to-list (fil ufile) #\.)) + (name (if (second name-list) + (first name-list) + (concatenate 'string (first name-list) (or extension "")))) + (type (when (second name-list) + (concatenate 'string (second name-list) (or extension ""))))) + (merge-pathnames + (make-pathname :name name :type type + :directory (cons :relative dirs)) + *umls-path*))) + (defun umls-pathname (filename &optional (extension "")) "Return pathname for a umls filename with an optional extension" (etypecase filename (string - (merge-pathnames - (make-pathname :name (concatenate 'string filename extension)) - (case (char filename 0) + (merge-pathnames + (make-pathname :name (concatenate 'string filename extension)) + (case (schar filename 0) ((#\M #\m) *meta-path*) ((#\L #\l) @@ -54,14 +74,8 @@ (t *umls-path*)))) (pathname - filename))) + filename))) -(defun read-umls-line (strm &optional (eof 'eof)) - "Read a line from a UMLS stream, split into fields" - (let ((line (read-line strm nil eof))) - (if (eq line eof) - eof - (delimited-string-to-list line #\| t)))) ;;; Find field lengths for LEX and NET files @@ -72,10 +86,10 @@ Currently, these are the LEX and NET files." (destructuring-bind (filename fields-max fields-av) length-list (let ((file (find-ufile filename))) (unless file - (error "Can't find ~A filename in ufiles")) + (error "Can't find ~A filename in ufiles" filename)) (unless (= (length fields-max) (length (fields file))) (error - "Number of file fields ~A not equal to field count in ufile ~S" + "Number of file fields ~A not equal to field count in ufile ~S" fields-max file)) (dotimes (i (length (fields file))) (declare (fixnum i)) @@ -86,39 +100,45 @@ Currently, these are the LEX and NET files." (setf (cmax col) (aref fields-max i)) (setf (av col) (aref fields-av i)) (ensure-ucol-datatype col (datatype-for-colname (col col))))))))) - + (defun ufiles-to-measure () "Returns a list of ufiles to measure" (loop for ufile in *umls-files* - unless (or (char= #\M (char (fil ufile) 0)) - (char= #\m (char (fil ufile) 0))) - collect ufile)) - - + unless (or (char= #\M (schar (fil ufile) 0)) + (char= #\m (schar (fil ufile) 0))) + collect ufile)) + + (defun ufiles-field-lengths (ufiles) "Returns a list of lists of containing (FILE MAX AV)" - (loop for ufile in ufiles collect (file-field-lengths (fil ufile)))) + (loop for ufile in ufiles collect (file-field-lengths ufile))) -(defun file-field-lengths (filename) +(defun file-field-lengths (ufile) "Returns a list of FILENAME MAX AV" + (declare (optimize (speed 3) (safety 0))) (let (fields-max fields-av num-fields (count-lines 0)) - (with-umls-file (line filename) + (with-umls-ufile (line ufile) (unless num-fields (setq num-fields (length line)) - (setq fields-max (make-array num-fields :element-type 'fixnum + (setq fields-max (make-array num-fields :element-type 'fixnum :initial-element 0)) (setq fields-av (make-array num-fields :element-type 'number :initial-element 0))) (dotimes (i num-fields) (declare (fixnum i)) - (let ((len (length (nth i line)))) + (let* ((str (nth i line)) + (len (length #-(and clisp unicode) str + #+(and clisp unicode) + (if *octet-sql-storage* + (ext:convert-string-to-bytes str charset:utf-8) + str)))) (incf (aref fields-av i) len) (when (> len (aref fields-max i)) (setf (aref fields-max i) len)))) (incf count-lines)) (dotimes (i num-fields) (setf (aref fields-av i) (float (/ (aref fields-av i) count-lines)))) - (list filename fields-max fields-av))) + (list (fil ufile) fields-max fields-av))) ;;; UMLS column/file functions @@ -137,7 +157,7 @@ Currently, these are the LEX and NET files." (defun make-ucol-for-column (colname filename ucols) ;; try to find column name without a terminal digit (let* ((len (length colname)) - (last-digit? (digit-char-p (char colname (1- len)))) + (last-digit? (digit-char-p (schar colname (1- len)))) (base-colname (if last-digit? (subseq colname 0 (1- len)) colname)) @@ -176,8 +196,12 @@ Currently, these are the LEX and NET files." (quote-str "'") (custom-value-fun)) (let ((ucol (make-instance 'ucol - :col col :des des :ref ref :min min :av av :max max :fil fil - :dty dty :sqltype sqltype :quote-str quote-str + :col col :des des :ref ref :min min :av av + :max (if (eql max 0) 1 max) ;; ensure at least one char wide + :fil fil + :dty dty + :sqltype sqltype + :quote-str quote-str :parse-fun (ensure-compiled-fun parse-fun) :custom-value-fun (ensure-compiled-fun custom-value-fun)))) (ensure-ucol-datatype ucol (datatype-for-colname col)) @@ -192,13 +216,16 @@ Currently, these are the LEX and NET files." (ensure-col-in-columns colname filename *umls-cols*)) (defun find-ufile (filename) - "Returns umls-file structure for a filename" + "Returns umls-file structure for a filename" (find-if #'(lambda (f) (string-equal filename (fil f))) *umls-files*)) (defun find-ucols-for-ufile (ufile) "Returns list of umls-cols for a file structure" (loop for colname in (fields ufile) - collect (find-ucol colname (fil ufile)))) + collect (find-ucol colname + (if (subdir ufile) + (concatenate 'string (subdir ufile) "/" (fil ufile)) + (fil ufile))))) (defun umls-field-string-to-list (fmt) "Converts a comma delimited list of fields into a list of field names. Will @@ -215,44 +242,100 @@ append a unique number (starting at 2) onto a column name that is repeated in th (setf (gethash colname col-counts) 1) colname)))))) -(defun make-ufile (fil des table cls rws bts fields) - (let ((ufile (make-instance 'ufile :fil fil :des des :table table :cls cls - :rws rws :bts bts :fields fields))) - (setf (ucols ufile) (find-ucols-for-ufile ufile)) - ufile)) +(defun decompose-fil (fil) + (if fil + (let ((pos (position #\/ fil))) + (if pos + (values (subseq fil (1+ pos)) (subseq fil 0 pos)) + (values fil nil))) + (values nil nil))) + +(defun filename-to-tablename (file) + (let ((pos (search ".RRF" file))) + (when pos + (setf file (subseq file 0 pos)))) + (substitute #\_ #\. file)) + +(defun make-ufile (dir fil des cls rws bts fields) + (multiple-value-bind (file subdir) (decompose-fil fil) + (let ((ufile (make-instance 'ufile :dir dir :fil file :subdir subdir + :des des :cls cls + :rws rws :bts bts :fields fields + :table (filename-to-tablename file)))) + (setf (ucols ufile) (find-ucols-for-ufile ufile)) + ufile))) (defun datatype-for-colname (colname) -"Return datatype for column name" +"Return datatype for column name" (second (find colname +col-datatypes+ :key #'car :test #'string-equal))) +(defun canonicalize-column-type (type) + (cond + ((string-equal type "SMALLINT") + (case *umls-sql-type* + (:mysql "SMALLINT") + ((:postgresql :postgresql-socket) "INT2") + (:oracle "NUMBER(5,0)") + (t "INTEGER"))) + ((string-equal type "INTEGER") + (case *umls-sql-type* + (:mysql "INTEGER") + ((:postgresql :postgresql-socket) "INT4") + (:oracle "NUMBER(9,0)") + (t "INTEGER"))) + ((string-equal type "BIGINT") + (case *umls-sql-type* + (:mysql "BIGINT") + ((:postgresql :postgresql-socket) "INT8") + (:oracle "NUMBER(38,0)") + (t "INTEGER"))) + ((string-equal type "TEXT") + (case *umls-sql-type* + (:mysql "TEXT") + ((:postgresql :postgresql-socket) "TEXT") + (:oracle "VARCHAR2(3000)") + (t "VARCHAR(3000)"))) + ((string-equal type "VARCHAR") + (case *umls-sql-type* + (:mysql "VARCHAR") + ((:postgresql :postgresql-socket) "VARCHAR") + (:oracle "VARCHAR2") + (t "VARCHAR"))) + ((string-equal type "NUMERIC") + (case *umls-sql-type* + (:mysql "NUMERIC") + ((:postgresql :postgresql-socket) "NUMERIC") + (:oracle "NUMBER") + (t "NUMERIC"))) + (t + type))) + (defun ensure-ucol-datatype (col datatype) -"Add data type information to column" + "Add data type information to column" (setf (datatype col) datatype) (case datatype - (sql-u (setf (sqltype col) "INTEGER" + (sql-u (setf (sqltype col) (canonicalize-column-type "INTEGER") (parse-fun col) #'parse-ui (quote-str col) "")) - (sql-s (setf (sqltype col) "SMALLINT" + (sql-s (setf (sqltype col) (canonicalize-column-type "SMALLINT") (parse-fun col) #'parse-integer (quote-str col) "")) - (sql-l (setf (sqltype col) "BIGINT" + (sql-l (setf (sqltype col) (canonicalize-column-type "BIGINT") (parse-fun col) #'parse-integer (quote-str col) "")) - (sql-i (setf (sqltype col) "INTEGER" + (sql-i (setf (sqltype col) (canonicalize-column-type "INTEGER") (parse-fun col) #'parse-integer (quote-str col) "")) - (sql-f (setf (sqltype col) "NUMERIC" + (sql-f (setf (sqltype col) (canonicalize-column-type "NUMERIC") (parse-fun col) #'read-from-string (quote-str col) "")) - (t ; Default column type, optimized text storage - (setf (parse-fun col) #'add-sql-quotes + (t ; Default column type, optimized text storage + (setf (parse-fun col) #'add-sql-quotes (quote-str col) "'") (when (and (cmax col) (av col)) (if (> (cmax col) 255) - (setf (sqltype col) "TEXT") - (if (< (- (cmax col) (av col)) 4) - (setf (sqltype col) "CHAR") ; if average bytes wasted < 4 - (setf (sqltype col) "VARCHAR"))))))) + (setf (sqltype col) (canonicalize-column-type "TEXT")) + (setf (sqltype col) (canonicalize-column-type "VARCHAR"))))))) (defun escape-column-name (name) (substitute #\_ #\/ name))