X-Git-Url: http://git.kpe.io/?p=umlisp-orf.git;a=blobdiff_plain;f=parse-common.lisp;fp=parse-common.lisp;h=5931b0583dd539a84237ffdbaa5e68eb87d79219;hp=0000000000000000000000000000000000000000;hb=d8fe27c58aa49f4a19f8b0dc11f97e0db7662e9e;hpb=020186ffddfabdeb617ec0fca3dec958dddce961

diff --git a/parse-common.lisp b/parse-common.lisp
new file mode 100644
index 0000000..5931b05
--- /dev/null
+++ b/parse-common.lisp
@@ -0,0 +1,256 @@
+;;;; -*- Mode: Lisp; Syntax: ANSI-Common-Lisp; Base: 10; Package: umlisp -*-
+;;;; *************************************************************************
+;;;; FILE IDENTIFICATION
+;;;;
+;;;; Name:     parse-common.lisp
+;;;; Purpose:  Common, stable parsing routines for UMLisp
+;;;; Author:   Kevin M. Rosenberg
+;;;; Created:  Apr 2000
+;;;;
+;;;; $Id$
+;;;;
+;;;; This file, part of UMLisp, is
+;;;;    Copyright (c) 2000-2004 by Kevin M. Rosenberg, M.D.
+;;;;
+;;;; UMLisp users are granted the rights to distribute and use this software
+;;;; as governed by the terms of the GNU General Public License.
+;;;; *************************************************************************
+
+(in-package #:umlisp-orf)
+
+(defun ensure-ucols+ufiles (&optional (alwaysclear nil))
+"Initialize all UMLS file and column structures if not already initialized"
+  (when (or alwaysclear (null *umls-files*))
+    (gen-ucols)
+    (gen-ufiles)
+    (ensure-field-lengths)))
+
+(defun add-ucols (ucols)
+  "Adds a ucol or list of ucols to *umls-cols*. Returns input value."
+  (setq *umls-cols* (append (mklist ucols) *umls-cols*))
+  ucols)
+
+(defun add-ufiles (ufiles)
+  "Adds a ufile or list of ufiles to *umls-filess*. Returns input value."
+  (setq *umls-files* (append (mklist ufiles) *umls-files*))
+  ufiles)
+
+(defun umls-pathname (filename &optional (extension ""))
+"Return pathname for a umls filename with an optional extension"
+  (etypecase filename
+    (string
+     (merge-pathnames 
+      (make-pathname :name (concatenate 'string filename extension)) 
+      (case (schar filename 0)
+	((#\M #\m)
+	 *meta-path*)
+	((#\L #\l)
+	 *lex-path*)
+	((#\S #\s)
+	 *net-path*)
+	(t
+	 *umls-path*))))
+    (pathname
+      filename)))
+
+(defun read-umls-line (strm &optional (eof 'eof))
+  "Read a line from a UMLS stream, split into fields"
+  (let ((line (read-line strm nil eof)))
+    (if (eq line eof)
+	eof
+	(delimited-string-to-list line #\| t))))
+
+;;; Find field lengths for LEX and NET files
+
+(defun ensure-field-lengths ()
+  "Initial colstruct field lengths for files that don't have a measurement.
+Currently, these are the LEX and NET files."
+  (dolist (length-list (ufiles-field-lengths (ufiles-to-measure)))
+    (destructuring-bind (filename fields-max fields-av) length-list
+      (let ((file (find-ufile filename)))
+	(unless file
+	  (error "Can't find ~A filename in ufiles" filename))
+	(unless (= (length fields-max) (length (fields file)))
+	  (error
+	   "Number of file fields ~A not equal to field count in ufile ~S" 
+	   fields-max file))
+	(dotimes (i (length (fields file)))
+	  (declare (fixnum i))
+	  (let* ((field (nth i (fields file)))
+		 (col (find-ucol field filename)))
+	    (unless col
+		(error "can't find column ~A" field))
+	    (setf (cmax col) (aref fields-max i))
+	    (setf (av col) (aref fields-av i))
+	    (ensure-ucol-datatype col (datatype-for-colname (col col)))))))))
+  
+(defun ufiles-to-measure ()
+  "Returns a list of ufiles to measure"
+  (loop for ufile in *umls-files*
+	unless (or (char= #\M (schar (fil ufile) 0))
+		   (char= #\m (schar (fil ufile) 0)))
+	collect ufile))
+    
+  
+(defun ufiles-field-lengths (ufiles)
+  "Returns a list of lists of containing (FILE MAX AV)"
+  (loop for ufile in ufiles collect (file-field-lengths (fil ufile))))
+
+(defun file-field-lengths (filename)
+  "Returns a list of FILENAME MAX AV"
+  (declare (optimize (speed 3) (safety 0)))
+  (let (fields-max fields-av num-fields (count-lines 0))
+    (with-umls-file (line filename)
+      (unless num-fields
+	(setq num-fields (length line))
+	(setq fields-max (make-array num-fields :element-type 'fixnum 
+				     :initial-element 0))
+	(setq fields-av (make-array num-fields :element-type 'number
+				    :initial-element 0)))
+      (dotimes (i num-fields)
+	(declare (fixnum i))
+	(let ((len (length (nth i line))))
+	  (incf (aref fields-av i) len)
+	  (when (> len (aref fields-max i))
+	    (setf (aref fields-max i) len))))
+      (incf count-lines))
+    (dotimes (i num-fields)
+      (setf (aref fields-av i) (float (/ (aref fields-av i) count-lines))))
+    (list filename fields-max fields-av)))
+
+;;; UMLS column/file functions
+
+(defun find-ucol-of-colname (colname filename ucols)
+"Returns list of umls-col structure for a column name and a filename"
+  (dolist (ucol ucols nil)
+    (when (and (string-equal filename (fil ucol))
+	       (string-equal colname (col ucol)))
+      (return-from find-ucol-of-colname ucol))))
+
+(defun ensure-col-in-columns (colname filename ucols)
+  (aif (find-ucol-of-colname colname filename ucols)
+       it
+       (add-ucols (make-ucol-for-column colname filename ucols))))
+
+(defun make-ucol-for-column (colname filename ucols)
+  ;; try to find column name without a terminal digit
+  (let* ((len (length colname))
+	 (last-digit? (digit-char-p (schar colname (1- len))))
+	 (base-colname (if last-digit?
+			   (subseq colname 0 (1- len))
+			   colname))
+	 (ucol (when last-digit?
+		 (find-ucol-of-colname base-colname filename ucols))))
+    (when (and last-digit? (null ucol))
+      (error "Couldn't find a base column for col ~A in file ~A"
+	     colname filename))
+    (copy-or-new-ucol colname filename ucol)))
+
+(defun copy-or-new-ucol (colname filename ucol)
+  (if ucol
+      (make-instance
+       'ucol
+       :col (copy-seq colname) :des (copy-seq (des ucol)) :ref (copy-seq (ref ucol))
+       :min (cmin ucol) :max (cmax ucol) :fil (copy-seq (fil ucol))
+       :sqltype (copy-seq (sqltype ucol)) :dty (copy-seq (dty ucol))
+       :parse-fun (parse-fun ucol) :quote-str (copy-seq (quote-str ucol))
+       :datatype (datatype ucol) :custom-value-fun (custom-value-fun ucol))
+      (make-empty-ucol colname filename)))
+
+(defun ensure-compiled-fun (fun)
+  "Ensure that a function is compiled"
+  (etypecase fun
+    (null
+     nil)
+    (function
+     (if (compiled-function-p fun)
+	 fun
+	 (compile nil fun)))
+    (list
+     (compile nil fun))))
+
+(defun make-ucol (col des ref min av max fil dty
+		  &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes)
+		  (quote-str "'") (custom-value-fun))
+  (let ((ucol (make-instance
+	       'ucol
+	       :col col :des des :ref ref :min min :av av 
+	       :max (if (eql max 0) 1 max) ;; ensure at least one char wide
+	       :fil fil
+	       :dty dty :sqltype sqltype :quote-str quote-str
+	       :parse-fun (ensure-compiled-fun parse-fun)
+ 	       :custom-value-fun (ensure-compiled-fun custom-value-fun))))
+    (ensure-ucol-datatype ucol (datatype-for-colname col))
+    ucol))
+
+(defun make-empty-ucol (colname filename)
+  ;;(format "call in make-empty-ucol: ~A/~A" colname filename)
+  (make-ucol (copy-seq colname) "Unknown" "" nil nil nil filename nil))
+
+(defun find-ucol (colname filename)
+  "Returns list of umls-col structure for a column name and a filename"
+  (ensure-col-in-columns colname filename *umls-cols*))
+
+(defun find-ufile (filename)
+  "Returns umls-file structure for a filename"  
+  (find-if #'(lambda (f) (string-equal filename (fil f))) *umls-files*))
+
+(defun find-ucols-for-ufile (ufile)
+  "Returns list of umls-cols for a file structure"
+  (loop for colname in (fields ufile)
+	collect (find-ucol colname (fil ufile))))
+
+(defun umls-field-string-to-list (fmt)
+  "Converts a comma delimited list of fields into a list of field names. Will
+append a unique number (starting at 2) onto a column name that is repeated in the list"
+  (let ((col-counts (make-hash-table :test 'equal)))
+    (loop for colname in (delimited-string-to-list (escape-column-name fmt) #\,)
+	  collect
+	  (multiple-value-bind (value found) (gethash colname col-counts)
+	    (cond
+	      (found
+		(incf (gethash colname col-counts))
+		(concatenate 'string colname (write-to-string (1+ value))))
+	      (t
+	       (setf (gethash colname col-counts) 1)
+	       colname))))))
+
+(defun make-ufile (fil des table cls rws bts fields)
+  (let ((ufile (make-instance 'ufile :fil fil :des des :table table :cls cls
+			      :rws rws :bts bts :fields fields)))
+    (setf (ucols ufile) (find-ucols-for-ufile ufile))
+    ufile))
+
+(defun datatype-for-colname (colname)
+"Return datatype for column name"  
+  (second (find colname +col-datatypes+ :key #'car :test #'string-equal)))
+
+(defun ensure-ucol-datatype (col datatype)
+"Add data type information to column"
+  (setf (datatype col) datatype)
+  (case datatype
+    (sql-u (setf (sqltype col) "INTEGER"
+		 (parse-fun col) #'parse-ui
+		 (quote-str col) ""))
+    (sql-s (setf (sqltype col) "SMALLINT" 
+		 (parse-fun col) #'parse-integer
+		 (quote-str col) ""))
+    (sql-l (setf (sqltype col) "BIGINT" 
+		 (parse-fun col) #'parse-integer
+		 (quote-str col) ""))
+    (sql-i (setf (sqltype col) "INTEGER" 
+		 (parse-fun col) #'parse-integer
+		 (quote-str col) ""))
+    (sql-f (setf (sqltype col) "NUMERIC" 
+		 (parse-fun col) #'read-from-string
+		 (quote-str col) ""))
+    (t			     ; Default column type, optimized text storage
+     (setf (parse-fun col) #'add-sql-quotes 
+	   (quote-str col) "'")
+     (when (and (cmax col) (av col))
+       (if (> (cmax col) 255)
+	   (setf (sqltype col) "TEXT")
+	   (setf (sqltype col) "VARCHAR"))))))
+
+(defun escape-column-name (name)
+  (substitute #\_ #\/ name))