+++ /dev/null
-;;;; -*- Mode: Lisp; Syntax: ANSI-Common-Lisp; Base: 10; Package: umlisp -*-
-;;;; *************************************************************************
-;;;; FILE IDENTIFICATION
-;;;;
-;;;; Name: parse-common.lisp
-;;;; Purpose: Common, stable parsing routines for UMLisp
-;;;; Author: Kevin M. Rosenberg
-;;;; Created: Apr 2000
-;;;;
-;;;; $Id$
-;;;;
-;;;; This file, part of UMLisp, is
-;;;; Copyright (c) 2000-2004 by Kevin M. Rosenberg, M.D.
-;;;;
-;;;; UMLisp users are granted the rights to distribute and use this software
-;;;; as governed by the terms of the GNU General Public License.
-;;;; *************************************************************************
-
-(in-package #:umlisp-orf)
-
-(defun ensure-ucols+ufiles (&optional (alwaysclear nil))
-"Initialize all UMLS file and column structures if not already initialized"
- (when (or alwaysclear (null *umls-files*))
- (gen-ucols)
- (gen-ufiles)
- (ensure-field-lengths)))
-
-(defun add-ucols (ucols)
- "Adds a ucol or list of ucols to *umls-cols*. Returns input value."
- (setq *umls-cols* (append (mklist ucols) *umls-cols*))
- ucols)
-
-(defun add-ufiles (ufiles)
- "Adds a ufile or list of ufiles to *umls-filess*. Returns input value."
- (setq *umls-files* (append (mklist ufiles) *umls-files*))
- ufiles)
-
-(defun umls-pathname (filename &optional (extension ""))
-"Return pathname for a umls filename with an optional extension"
- (etypecase filename
- (string
- (merge-pathnames
- (make-pathname :name (concatenate 'string filename extension))
- (case (schar filename 0)
- ((#\M #\m)
- *meta-path*)
- ((#\L #\l)
- *lex-path*)
- ((#\S #\s)
- *net-path*)
- (t
- *umls-path*))))
- (pathname
- filename)))
-
-(defun read-umls-line (strm &optional (eof 'eof))
- "Read a line from a UMLS stream, split into fields"
- (let ((line (read-line strm nil eof)))
- (if (eq line eof)
- eof
- (delimited-string-to-list line #\| t))))
-
-;;; Find field lengths for LEX and NET files
-
-(defun ensure-field-lengths ()
- "Initial colstruct field lengths for files that don't have a measurement.
-Currently, these are the LEX and NET files."
- (dolist (length-list (ufiles-field-lengths (ufiles-to-measure)))
- (destructuring-bind (filename fields-max fields-av) length-list
- (let ((file (find-ufile filename)))
- (unless file
- (error "Can't find ~A filename in ufiles" filename))
- (unless (= (length fields-max) (length (fields file)))
- (error
- "Number of file fields ~A not equal to field count in ufile ~S"
- fields-max file))
- (dotimes (i (length (fields file)))
- (declare (fixnum i))
- (let* ((field (nth i (fields file)))
- (col (find-ucol field filename)))
- (unless col
- (error "can't find column ~A" field))
- (setf (cmax col) (aref fields-max i))
- (setf (av col) (aref fields-av i))
- (ensure-ucol-datatype col (datatype-for-colname (col col)))))))))
-
-(defun ufiles-to-measure ()
- "Returns a list of ufiles to measure"
- (loop for ufile in *umls-files*
- unless (or (char= #\M (schar (fil ufile) 0))
- (char= #\m (schar (fil ufile) 0)))
- collect ufile))
-
-
-(defun ufiles-field-lengths (ufiles)
- "Returns a list of lists of containing (FILE MAX AV)"
- (loop for ufile in ufiles collect (file-field-lengths (fil ufile))))
-
-(defun file-field-lengths (filename)
- "Returns a list of FILENAME MAX AV"
- (declare (optimize (speed 3) (safety 0)))
- (let (fields-max fields-av num-fields (count-lines 0))
- (with-umls-file (line filename)
- (unless num-fields
- (setq num-fields (length line))
- (setq fields-max (make-array num-fields :element-type 'fixnum
- :initial-element 0))
- (setq fields-av (make-array num-fields :element-type 'number
- :initial-element 0)))
- (dotimes (i num-fields)
- (declare (fixnum i))
- (let ((len (length (nth i line))))
- (incf (aref fields-av i) len)
- (when (> len (aref fields-max i))
- (setf (aref fields-max i) len))))
- (incf count-lines))
- (dotimes (i num-fields)
- (setf (aref fields-av i) (float (/ (aref fields-av i) count-lines))))
- (list filename fields-max fields-av)))
-
-;;; UMLS column/file functions
-
-(defun find-ucol-of-colname (colname filename ucols)
-"Returns list of umls-col structure for a column name and a filename"
- (dolist (ucol ucols nil)
- (when (and (string-equal filename (fil ucol))
- (string-equal colname (col ucol)))
- (return-from find-ucol-of-colname ucol))))
-
-(defun ensure-col-in-columns (colname filename ucols)
- (aif (find-ucol-of-colname colname filename ucols)
- it
- (add-ucols (make-ucol-for-column colname filename ucols))))
-
-(defun make-ucol-for-column (colname filename ucols)
- ;; try to find column name without a terminal digit
- (let* ((len (length colname))
- (last-digit? (digit-char-p (schar colname (1- len))))
- (base-colname (if last-digit?
- (subseq colname 0 (1- len))
- colname))
- (ucol (when last-digit?
- (find-ucol-of-colname base-colname filename ucols))))
- (when (and last-digit? (null ucol))
- (error "Couldn't find a base column for col ~A in file ~A"
- colname filename))
- (copy-or-new-ucol colname filename ucol)))
-
-(defun copy-or-new-ucol (colname filename ucol)
- (if ucol
- (make-instance
- 'ucol
- :col (copy-seq colname) :des (copy-seq (des ucol)) :ref (copy-seq (ref ucol))
- :min (cmin ucol) :max (cmax ucol) :fil (copy-seq (fil ucol))
- :sqltype (copy-seq (sqltype ucol)) :dty (copy-seq (dty ucol))
- :parse-fun (parse-fun ucol) :quote-str (copy-seq (quote-str ucol))
- :datatype (datatype ucol) :custom-value-fun (custom-value-fun ucol))
- (make-empty-ucol colname filename)))
-
-(defun ensure-compiled-fun (fun)
- "Ensure that a function is compiled"
- (etypecase fun
- (null
- nil)
- (function
- (if (compiled-function-p fun)
- fun
- (compile nil fun)))
- (list
- (compile nil fun))))
-
-(defun make-ucol (col des ref min av max fil dty
- &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes)
- (quote-str "'") (custom-value-fun))
- (let ((ucol (make-instance
- 'ucol
- :col col :des des :ref ref :min min :av av
- :max (if (eql max 0) 1 max) ;; ensure at least one char wide
- :fil fil
- :dty dty :sqltype sqltype :quote-str quote-str
- :parse-fun (ensure-compiled-fun parse-fun)
- :custom-value-fun (ensure-compiled-fun custom-value-fun))))
- (ensure-ucol-datatype ucol (datatype-for-colname col))
- ucol))
-
-(defun make-empty-ucol (colname filename)
- ;;(format "call in make-empty-ucol: ~A/~A" colname filename)
- (make-ucol (copy-seq colname) "Unknown" "" nil nil nil filename nil))
-
-(defun find-ucol (colname filename)
- "Returns list of umls-col structure for a column name and a filename"
- (ensure-col-in-columns colname filename *umls-cols*))
-
-(defun find-ufile (filename)
- "Returns umls-file structure for a filename"
- (find-if #'(lambda (f) (string-equal filename (fil f))) *umls-files*))
-
-(defun find-ucols-for-ufile (ufile)
- "Returns list of umls-cols for a file structure"
- (loop for colname in (fields ufile)
- collect (find-ucol colname (fil ufile))))
-
-(defun umls-field-string-to-list (fmt)
- "Converts a comma delimited list of fields into a list of field names. Will
-append a unique number (starting at 2) onto a column name that is repeated in the list"
- (let ((col-counts (make-hash-table :test 'equal)))
- (loop for colname in (delimited-string-to-list (escape-column-name fmt) #\,)
- collect
- (multiple-value-bind (value found) (gethash colname col-counts)
- (cond
- (found
- (incf (gethash colname col-counts))
- (concatenate 'string colname (write-to-string (1+ value))))
- (t
- (setf (gethash colname col-counts) 1)
- colname))))))
-
-(defun make-ufile (fil des table cls rws bts fields)
- (let ((ufile (make-instance 'ufile :fil fil :des des :table table :cls cls
- :rws rws :bts bts :fields fields)))
- (setf (ucols ufile) (find-ucols-for-ufile ufile))
- ufile))
-
-(defun datatype-for-colname (colname)
-"Return datatype for column name"
- (second (find colname +col-datatypes+ :key #'car :test #'string-equal)))
-
-(defun ensure-ucol-datatype (col datatype)
-"Add data type information to column"
- (setf (datatype col) datatype)
- (case datatype
- (sql-u (setf (sqltype col) "INTEGER"
- (parse-fun col) #'parse-ui
- (quote-str col) ""))
- (sql-s (setf (sqltype col) "SMALLINT"
- (parse-fun col) #'parse-integer
- (quote-str col) ""))
- (sql-l (setf (sqltype col) "BIGINT"
- (parse-fun col) #'parse-integer
- (quote-str col) ""))
- (sql-i (setf (sqltype col) "INTEGER"
- (parse-fun col) #'parse-integer
- (quote-str col) ""))
- (sql-f (setf (sqltype col) "NUMERIC"
- (parse-fun col) #'read-from-string
- (quote-str col) ""))
- (t ; Default column type, optimized text storage
- (setf (parse-fun col) #'add-sql-quotes
- (quote-str col) "'")
- (when (and (cmax col) (av col))
- (if (> (cmax col) 255)
- (setf (sqltype col) "TEXT")
- (setf (sqltype col) "VARCHAR"))))))
-
-(defun escape-column-name (name)
- (substitute #\_ #\/ name))
projects / umlisp.git / blobdiff