1 ;;;; -*- Mode: Lisp; Syntax: ANSI-Common-Lisp; Base: 10; Package: umlisp -*-
2 ;;;; *************************************************************************
3 ;;;; FILE IDENTIFICATION
5 ;;;; Name: parse-common.lisp
6 ;;;; Purpose: Common, stable parsing routines for UMLisp
7 ;;;; Author: Kevin M. Rosenberg
10 ;;;; This file, part of UMLisp, is
11 ;;;; Copyright (c) 2000-2010 by Kevin M. Rosenberg, M.D.
13 ;;;; UMLisp users are granted the rights to distribute and use this software
14 ;;;; as governed by the terms of the GNU General Public License.
15 ;;;; *************************************************************************
19 (defun ensure-ucols+ufiles (&optional (alwaysclear nil))
20 "Initialize all UMLS file and column structures if not already initialized"
22 (when (or alwaysclear (null *umls-files*))
23 (setf *umls-cols* nil *umls-files* nil)
26 (set-ucols-for-ufiles *umls-files*)
27 (ensure-field-lengths))
29 (warn "Error reading ucols+ufiles: ~A." e)
30 (setf *umls-cols* nil *umls-files* nil)
35 (defun add-ucols (ucols)
36 "Adds a ucol or list of ucols to *umls-cols*. Returns input value."
37 (setq *umls-cols* (append (mklist ucols) *umls-cols*))
40 (defun add-ufiles (ufiles)
41 "Adds a ufile or list of ufiles to *umls-filess*. Returns input value."
42 (setq *umls-files* (append (mklist ufiles) *umls-files*))
45 (defun ufile-pathname (ufile &optional (extension ""))
46 "Return pathname for a umls filename with an optional extension"
47 (assert (typep ufile 'ufile))
48 (let* ((dirs (nconc (list (dir ufile))
49 (awhen (subdir ufile) (list it))))
50 (name-list (delimited-string-to-list (fil ufile) #\.))
51 (name (if (second name-list)
53 (concatenate 'string (first name-list) (or extension ""))))
54 (type (when (second name-list)
55 (concatenate 'string (second name-list) (or extension "")))))
57 (make-pathname :name name :type type
58 :directory (cons :relative dirs))
61 (defun umls-pathname (filename &optional (extension ""))
62 "Return pathname for a umls filename with an optional extension"
65 (let* ((name-list (delimited-string-to-list filename #\.))
66 (name (if (second name-list)
68 (concatenate 'string (first name-list) (or extension ""))))
69 (type (when (second name-list)
70 (concatenate 'string (second name-list) (or extension "")))))
72 (make-pathname :name name :type type)
73 (case (schar filename 0)
86 ;;; Find field lengths for LEX and NET files
88 (defun ensure-field-lengths ()
89 "Initial colstruct field lengths for files that don't have a measurement.
90 Currently, these are the LEX and NET files."
91 (dolist (length-list (ufiles-field-lengths (ufiles-to-measure)))
92 (destructuring-bind (filename fields-max fields-av) length-list
93 (let ((file (find-ufile filename)))
95 (error "Can't find ~A filename in ufiles" filename))
96 (unless (= (length fields-max) (length (fields file)))
98 "Number of file fields ~A not equal to field count in ufile ~S"
100 (dotimes (i (length (fields file)))
102 (let* ((field (nth i (fields file)))
103 (col (find-ucol field filename)))
105 (error "can't find column ~A" field))
106 (setf (cmax col) (aref fields-max i))
107 (setf (av col) (aref fields-av i))
108 (ensure-ucol-datatype col (datatype-for-colname (col col)))))))))
110 (defun ufiles-to-measure ()
111 "Returns a list of ufiles to measure"
112 (loop for ufile in *umls-files*
113 unless (or (char= #\M (schar (fil ufile) 0))
114 (char= #\m (schar (fil ufile) 0)))
118 (defun ufiles-field-lengths (ufiles)
119 "Returns a list of lists of containing (FILE MAX AV)"
120 (loop for ufile in ufiles collect (file-field-lengths ufile)))
122 (defun file-field-lengths (ufile)
123 "Returns a list of FILENAME MAX AV"
124 (declare (optimize (speed 3) (safety 0)))
125 (let (fields-max fields-av num-fields (count-lines 0))
126 (declare (fixnum count-lines))
127 (with-umls-ufile (line ufile)
129 (setq num-fields (length line))
130 (setq fields-max (make-array num-fields :element-type 'fixnum
132 (setq fields-av (make-array num-fields :element-type '(or integer float)
133 :initial-element 0)))
134 (dotimes (i num-fields)
136 (let* ((str (nth i line))
137 (len (length #-(and clisp unicode) str
138 #+(and clisp unicode)
139 (if *octet-sql-storage*
140 (ext:convert-string-to-bytes str charset:utf-8)
142 #-(and clisp unicode) (declare (string str))
143 (declare (type (integer 0 100000000) len))
144 (incf (aref fields-av i) len)
145 (when (> len (aref fields-max i))
146 (setf (aref fields-max i) len))))
148 (dotimes (i num-fields)
149 (setf (aref fields-av i) (float (/ (aref fields-av i) count-lines))))
150 (list (fil ufile) fields-max fields-av)))
152 ;;; UMLS column/file functions
154 (defun find-ucol-of-colname (colname filename ucols)
155 "Returns list of umls-col structure for a column name and a filename"
156 (dolist (ucol ucols nil)
157 (when (and (string-equal filename (fil ucol))
158 (string-equal colname (col ucol)))
159 (return-from find-ucol-of-colname ucol))))
161 (defun ensure-col-in-columns (colname filename ucols)
162 (aif (find-ucol-of-colname colname filename ucols)
164 (add-ucols (make-ucol-for-column colname filename ucols))))
166 (defun make-ucol-for-column (colname filename ucols)
167 ;; try to find column name without a terminal digit
168 (let* ((len (length colname))
169 (last-digit? (digit-char-p (schar colname (1- len))))
170 (base-colname (if last-digit?
171 (subseq colname 0 (1- len))
173 (ucol (when last-digit?
174 (find-ucol-of-colname base-colname filename ucols))))
175 (when (and last-digit? (null ucol))
176 (error "Couldn't find a base column for col ~A in file ~A"
178 (copy-or-new-ucol colname filename ucol)))
180 (defun copy-or-new-ucol (colname filename ucol)
184 :col (copy-seq colname) :des (copy-seq (des ucol)) :ref (copy-seq (ref ucol))
185 :min (cmin ucol) :max (cmax ucol) :fil (copy-seq (fil ucol))
186 :sqltype (copy-seq (sqltype ucol)) :dty (copy-seq (dty ucol))
187 :parse-fun (parse-fun ucol) :quote-str (copy-seq (quote-str ucol))
188 :datatype (datatype ucol) :custom-value-fun (custom-value-fun ucol))
189 (make-empty-ucol colname filename)))
191 (defun ensure-compiled-fun (fun)
192 "Ensure that a function is compiled"
197 (if (compiled-function-p fun)
203 (defun make-ucol (col des ref min av max fil dty
204 &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes)
205 (quote-str "'") (custom-value-fun))
206 (let ((ucol (make-instance
208 :col col :des des :ref ref :min min :av av
209 :max (if (eql max 0) 1 max) ;; ensure at least one char wide
214 :parse-fun (ensure-compiled-fun parse-fun)
215 :custom-value-fun (ensure-compiled-fun custom-value-fun))))
216 (ensure-ucol-datatype ucol (datatype-for-colname col))
219 (defun make-empty-ucol (colname filename)
220 ;;(format "call in make-empty-ucol: ~A/~A" colname filename)
221 (make-ucol (copy-seq colname) "Unknown" "" nil nil nil filename nil))
223 (defun find-ucol (colname filename)
224 "Returns list of umls-col structure for a column name and a filename"
225 (ensure-col-in-columns colname filename *umls-cols*))
227 (defun find-ufile (filename)
228 "Returns umls-file structure for a filename"
229 (find-if #'(lambda (f) (string= filename (fil f))) *umls-files*))
231 (defun position-field-file (filename fieldname)
232 "Returns the position of a field in a file"
233 (let ((ufile (find-ufile filename)))
235 (warn "Unable to find ufile for filename ~A." filename)
236 (return-from position-field-file nil))
237 (let ((pos (position fieldname (fields ufile) :test #'string=)))
239 (warn "Unable to find field ~A in ufile ~S." fieldname ufile)
240 (return-from position-field-file nil))
243 (defun find-ucols-for-ufile (ufile)
244 "Returns list of umls-cols for a file structure"
245 (loop for colname in (fields ufile)
246 collect (find-ucol colname
248 (concatenate 'string (subdir ufile) "/" (fil ufile))
251 (defun umls-field-string-to-list (fmt)
252 "Converts a comma delimited list of fields into a list of field names. Will
253 append a unique number (starting at 2) onto a column name that is repeated in the list"
254 (let ((col-counts (make-hash-table :test 'equal)))
255 (loop for colname in (delimited-string-to-list (escape-column-name fmt) #\,)
257 (multiple-value-bind (value found) (gethash colname col-counts)
260 (incf (gethash colname col-counts))
261 (concatenate 'string colname (write-to-string (1+ value))))
263 (setf (gethash colname col-counts) 1)
266 (defun decompose-fil (fil)
268 (let ((pos (position #\/ fil)))
270 (values (subseq fil (1+ pos)) (subseq fil 0 pos))
274 (defun filename-to-tablename (file)
275 (let ((pos (search ".RRF" file)))
277 (setf file (subseq file 0 pos))))
278 (substitute #\_ #\. file))
280 (defun make-ufile (dir fil des cls rws bts fields)
281 (multiple-value-bind (file subdir) (decompose-fil fil)
282 (let ((ufile (make-instance 'ufile :dir dir :fil file :subdir subdir
284 :rws rws :bts bts :fields fields
285 :table (filename-to-tablename file))))
288 (defun set-ucols-for-ufiles (ufiles)
289 (dolist (ufile ufiles)
290 (setf (ucols ufile) (find-ucols-for-ufile ufile))))
292 (defun datatype-for-colname (colname)
293 "Return datatype for column name"
294 (second (find colname +col-datatypes+ :key #'car :test #'string-equal)))
296 (defun canonicalize-column-type (type)
298 ((string-equal type "TINYINT")
299 (case *umls-sql-type*
301 ((:postgresql :postgresql-socket) "INT1")
302 (:oracle "NUMBER(3,0)")
304 ((string-equal type "SMALLINT")
305 (case *umls-sql-type*
307 ((:postgresql :postgresql-socket) "INT2")
308 (:oracle "NUMBER(5,0)")
310 ((string-equal type "INTEGER")
311 (case *umls-sql-type*
313 ((:postgresql :postgresql-socket) "INT4")
314 (:oracle "NUMBER(9,0)")
316 ((string-equal type "BIGINT")
317 (case *umls-sql-type*
319 ((:postgresql :postgresql-socket) "INT8")
320 (:oracle "NUMBER(38,0)")
322 ((string-equal type "TEXT")
323 (case *umls-sql-type*
325 ((:postgresql :postgresql-socket) "TEXT")
326 (:oracle "VARCHAR2(3000)")
327 (t "VARCHAR(3000)")))
328 ((string-equal type "VARCHAR")
329 (case *umls-sql-type*
331 ((:postgresql :postgresql-socket) "VARCHAR")
334 ((string-equal type "NUMERIC")
335 (case *umls-sql-type*
337 ((:postgresql :postgresql-socket) "NUMERIC")
343 (defun ensure-ucol-datatype (col datatype)
344 "Add data type information to column"
345 (setf (datatype col) datatype)
347 (sql-u (setf (sqltype col) (canonicalize-column-type "INTEGER")
348 (parse-fun col) #'parse-ui
350 (sql-s (setf (sqltype col) (canonicalize-column-type "SMALLINT")
351 (parse-fun col) #'parse-integer
353 (sql-l (setf (sqltype col) (canonicalize-column-type "BIGINT")
354 (parse-fun col) #'parse-integer
356 (sql-i (setf (sqltype col) (canonicalize-column-type "INTEGER")
357 (parse-fun col) #'parse-integer
359 (sql-t (setf (sqltype col) (canonicalize-column-type "TINYINT")
360 (parse-fun col) #'parse-integer
362 (sql-f (setf (sqltype col) (canonicalize-column-type "NUMERIC")
363 (parse-fun col) #'read-from-string
365 (t ; Default column type, optimized text storage
366 (setf (parse-fun col) #'add-sql-quotes
368 (when (and (cmax col) (av col))
369 (if (> (cmax col) 255)
370 (setf (sqltype col) (canonicalize-column-type "TEXT"))
371 (setf (sqltype col) (canonicalize-column-type "VARCHAR")))))))
373 (defun escape-column-name (name)
374 (substitute #\_ #\/ name))