1 ;;;; -*- Mode: Lisp; Syntax: ANSI-Common-Lisp; Base: 10; Package: umlisp -*-
2 ;;;; *************************************************************************
3 ;;;; FILE IDENTIFICATION
5 ;;;; Name: parse-common.lisp
6 ;;;; Purpose: Common, stable parsing routines for UMLisp
7 ;;;; Author: Kevin M. Rosenberg
8 ;;;; Date Started: Apr 2000
10 ;;;; $Id: parse-common.lisp,v 1.15 2003/06/10 22:30:16 kevin Exp $
12 ;;;; This file, part of UMLisp, is
13 ;;;; Copyright (c) 2000-2003 by Kevin M. Rosenberg, M.D.
15 ;;;; UMLisp users are granted the rights to distribute and use this software
16 ;;;; as governed by the terms of the GNU General Public License.
17 ;;;; *************************************************************************
21 (eval-when (:compile-toplevel)
22 (declaim (optimize (speed 3) (safety 1) (compilation-speed 0) (debug 3))))
24 (defun ensure-ucols+ufiles (&optional (alwaysclear nil))
25 "Initialize all UMLS file and column structures if not already initialized"
26 (when (or alwaysclear (null *umls-files*))
29 (ensure-field-lengths)))
31 (defun add-ucols (ucols)
32 "Adds a ucol or list of ucols to *umls-cols*. Returns input value."
33 (setq *umls-cols* (append (mklist ucols) *umls-cols*))
36 (defun add-ufiles (ufiles)
37 "Adds a ufile or list of ufiles to *umls-filess*. Returns input value."
38 (setq *umls-files* (append (mklist ufiles) *umls-files*))
41 (defun umls-pathname (filename &optional (extension ""))
42 "Return pathname for a umls filename with an optional extension"
46 (make-pathname :name (concatenate 'string filename extension))
47 (case (char filename 0)
59 (defun read-umls-line (strm &optional (eof 'eof))
60 "Read a line from a UMLS stream, split into fields"
61 (let ((line (read-line strm nil eof)))
64 (delimited-string-to-list line #\| t))))
66 ;;; Find field lengths for LEX and NET files
68 (defun ensure-field-lengths ()
69 "Initial colstruct field lengths for files that don't have a measurement.
70 Currently, these are the LEX and NET files."
71 (dolist (length-list (ufiles-field-lengths (ufiles-to-measure)))
72 (destructuring-bind (filename fields-max fields-av) length-list
73 (let ((file (find-ufile filename)))
75 (error "Can't find ~A filename in ufiles"))
76 (unless (= (length fields-max) (length (fields file)))
78 "Number of file fields ~A not equal to field count in ufile ~S"
80 (dotimes (i (length (fields file)))
82 (let* ((field (nth i (fields file)))
83 (col (find-ucol field filename)))
85 (error "can't find column ~A" field))
86 (setf (cmax col) (aref fields-max i))
87 (setf (av col) (aref fields-av i))
88 (ensure-ucol-datatype col (datatype-for-colname (col col)))))))))
90 (defun ufiles-to-measure ()
91 "Returns a list of ufiles to measure"
92 (loop for ufile in *umls-files*
93 unless (or (char= #\M (char (fil ufile) 0))
94 (char= #\m (char (fil ufile) 0)))
98 (defun ufiles-field-lengths (ufiles)
99 "Returns a list of lists of containing (FILE MAX AV)"
100 (loop for ufile in ufiles collect (file-field-lengths (fil ufile))))
102 (defun file-field-lengths (filename)
103 "Returns a list of FILENAME MAX AV"
104 (let (fields-max fields-av num-fields (count-lines 0))
105 (with-umls-file (line filename)
107 (setq num-fields (length line))
108 (setq fields-max (make-array num-fields :element-type 'fixnum
110 (setq fields-av (make-array num-fields :element-type 'number
111 :initial-element 0)))
112 (dotimes (i num-fields)
114 (let ((len (length (nth i line))))
115 (incf (aref fields-av i) len)
116 (when (> len (aref fields-max i))
117 (setf (aref fields-max i) len))))
119 (dotimes (i num-fields)
120 (setf (aref fields-av i) (float (/ (aref fields-av i) count-lines))))
121 (list filename fields-max fields-av)))
123 ;;; UMLS column/file functions
125 (defun find-ucol-of-colname (colname filename ucols)
126 "Returns list of umls-col structure for a column name and a filename"
127 (dolist (ucol ucols nil)
128 (when (and (string-equal filename (fil ucol))
129 (string-equal colname (col ucol)))
130 (return-from find-ucol-of-colname ucol))))
132 (defun ensure-col-in-columns (colname filename ucols)
133 (aif (find-ucol-of-colname colname filename ucols)
135 (add-ucols (make-ucol-for-column colname filename ucols))))
137 (defun make-ucol-for-column (colname filename ucols)
138 ;; try to find column name without a terminal digit
139 (let* ((len (length colname))
140 (last-digit? (digit-char-p (char colname (1- len))))
141 (base-colname (if last-digit?
142 (subseq colname 0 (1- len))
144 (ucol (when last-digit?
145 (find-ucol-of-colname base-colname filename ucols))))
146 (when (and last-digit? (null ucol))
147 (error "Couldn't find a base column for col ~A in file ~A"
149 (copy-or-new-ucol colname filename ucol)))
151 (defun copy-or-new-ucol (colname filename ucol)
155 :col (copy-seq colname) :des (copy-seq (des ucol)) :ref (copy-seq (ref ucol))
156 :min (cmin ucol) :max (cmax ucol) :fil (copy-seq (fil ucol))
157 :sqltype (copy-seq (sqltype ucol)) :dty (copy-seq (dty ucol))
158 :parse-fun (parse-fun ucol) :quote-str (copy-seq (quote-str ucol))
159 :datatype (datatype ucol) :custom-value-fun (custom-value-fun ucol))
160 (make-empty-ucol colname filename)))
162 (defun ensure-compiled-fun (fun)
163 "Ensure that a function is compiled"
168 (if (compiled-function-p fun)
174 (defun make-ucol (col des ref min av max fil dty
175 &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes)
176 (quote-str "'") (custom-value-fun))
177 (let ((ucol (make-instance
179 :col col :des des :ref ref :min min :av av :max max :fil fil
180 :dty dty :sqltype sqltype :quote-str quote-str
181 :parse-fun (ensure-compiled-fun parse-fun)
182 :custom-value-fun (ensure-compiled-fun custom-value-fun))))
183 (ensure-ucol-datatype ucol (datatype-for-colname col))
186 (defun make-empty-ucol (colname filename)
187 ;;(format "call in make-empty-ucol: ~A/~A" colname filename)
188 (make-ucol (copy-seq colname) "Unknown" "" nil nil nil filename nil))
190 (defun find-ucol (colname filename)
191 "Returns list of umls-col structure for a column name and a filename"
192 (ensure-col-in-columns colname filename *umls-cols*))
194 (defun find-ufile (filename)
195 "Returns umls-file structure for a filename"
196 (find-if #'(lambda (f) (string-equal filename (fil f))) *umls-files*))
198 (defun find-ucols-for-ufile (ufile)
199 "Returns list of umls-cols for a file structure"
200 (loop for colname in (fields ufile)
201 collect (find-ucol colname (fil ufile))))
203 (defun umls-field-string-to-list (fmt)
204 "Converts a comma delimited list of fields into a list of field names. Will
205 append a unique number (starting at 2) onto a column name that is repeated in the list"
206 (let ((col-counts (make-hash-table :test 'equal)))
207 (loop for colname in (delimited-string-to-list (escape-column-name fmt) #\,)
209 (multiple-value-bind (value found) (gethash colname col-counts)
212 (incf (gethash colname col-counts))
213 (concatenate 'string colname (write-to-string (1+ value))))
215 (setf (gethash colname col-counts) 1)
218 (defun make-ufile (fil des table cls rws bts fields)
219 (let ((ufile (make-instance 'ufile :fil fil :des des :table table :cls cls
220 :rws rws :bts bts :fields fields)))
221 (setf (ucols ufile) (find-ucols-for-ufile ufile))
224 (defun datatype-for-colname (colname)
225 "Return datatype for column name"
226 (second (find colname +col-datatypes+ :key #'car :test #'string-equal)))
228 (defun ensure-ucol-datatype (col datatype)
229 "Add data type information to column"
230 (setf (datatype col) datatype)
232 (sql-u (setf (sqltype col) "INTEGER"
233 (parse-fun col) #'parse-ui
235 (sql-s (setf (sqltype col) "SMALLINT"
236 (parse-fun col) #'parse-integer
238 (sql-l (setf (sqltype col) "BIGINT"
239 (parse-fun col) #'parse-integer
241 (sql-i (setf (sqltype col) "INTEGER"
242 (parse-fun col) #'parse-integer
244 (sql-f (setf (sqltype col) "NUMERIC"
245 (parse-fun col) #'read-from-string
247 (t ; Default column type, optimized text storage
248 (setf (parse-fun col) #'add-sql-quotes
250 (when (and (cmax col) (av col))
251 (if (> (cmax col) 255)
252 (setf (sqltype col) "TEXT")
253 (if (< (- (cmax col) (av col)) 4)
254 (setf (sqltype col) "CHAR") ; if average bytes wasted < 4
255 (setf (sqltype col) "VARCHAR")))))))
257 (defun escape-column-name (name)
258 (substitute #\_ #\/ name))