;;;; *************************************************************************
;;;; FILE IDENTIFICATION
;;;;
-;;;; Name: parse-common.lisp
-;;;; Purpose: Common, stable parsing routines for UMLisp
-;;;; Author: Kevin M. Rosenberg
-;;;; Date Started: Apr 2000
+;;;; Name: parse-common.lisp
+;;;; Purpose: Common, stable parsing routines for UMLisp
+;;;; Author: Kevin M. Rosenberg
+;;;; Created: Apr 2000
;;;;
-;;;; $Id: parse-common.lisp,v 1.16 2003/08/26 04:46:16 kevin Exp $
+;;;; $Id$
;;;;
;;;; This file, part of UMLisp, is
-;;;; Copyright (c) 2000-2003 by Kevin M. Rosenberg, M.D.
+;;;; Copyright (c) 2000-2004 by Kevin M. Rosenberg, M.D.
;;;;
;;;; UMLisp users are granted the rights to distribute and use this software
;;;; as governed by the terms of the GNU General Public License.
(in-package #:umlisp)
-(eval-when (:compile-toplevel)
- (declaim (optimize (speed 3) (safety 1) (compilation-speed 0) (debug 3))))
-
(defun ensure-ucols+ufiles (&optional (alwaysclear nil))
-"Initialize all UMLS file and column structures if not already initialized"
+ "Initialize all UMLS file and column structures if not already initialized"
(when (or alwaysclear (null *umls-files*))
+ (setq *umls-cols* nil)
+ (setq *umls-files* nil)
(gen-ucols)
(gen-ufiles)
(ensure-field-lengths)))
(setq *umls-files* (append (mklist ufiles) *umls-files*))
ufiles)
+(defun ufile-pathname (ufile &optional (extension ""))
+ "Return pathname for a umls filename with an optional extension"
+ (assert (typep ufile 'ufile))
+ (let ((dirs (append (list (dir ufile))
+ (awhen (subdir ufile) (list it)))))
+ (merge-pathnames
+ (make-pathname :name (concatenate 'string (fil ufile) extension)
+ :directory (cons :relative dirs))
+ *umls-path*)))
+
(defun umls-pathname (filename &optional (extension ""))
"Return pathname for a umls filename with an optional extension"
(etypecase filename
(string
(merge-pathnames
(make-pathname :name (concatenate 'string filename extension))
- (case (char filename 0)
+ (case (schar filename 0)
((#\M #\m)
*meta-path*)
((#\L #\l)
(t
*umls-path*))))
(pathname
- filename)))
+ filename)))
-(defun read-umls-line (strm &optional (eof 'eof))
- "Read a line from a UMLS stream, split into fields"
- (let ((line (read-line strm nil eof)))
- (if (eq line eof)
- eof
- (delimited-string-to-list line #\| t))))
;;; Find field lengths for LEX and NET files
(defun ufiles-to-measure ()
"Returns a list of ufiles to measure"
(loop for ufile in *umls-files*
- unless (or (char= #\M (char (fil ufile) 0))
- (char= #\m (char (fil ufile) 0)))
- collect ufile))
+ unless (or (char= #\M (schar (fil ufile) 0))
+ (char= #\m (schar (fil ufile) 0)))
+ collect ufile))
(defun ufiles-field-lengths (ufiles)
"Returns a list of lists of containing (FILE MAX AV)"
- (loop for ufile in ufiles collect (file-field-lengths (fil ufile))))
+ (loop for ufile in ufiles collect (file-field-lengths ufile)))
-(defun file-field-lengths (filename)
+(defun file-field-lengths (ufile)
"Returns a list of FILENAME MAX AV"
+ (declare (optimize (speed 3) (safety 0)))
(let (fields-max fields-av num-fields (count-lines 0))
- (with-umls-file (line filename)
+ (with-umls-ufile (line ufile)
(unless num-fields
(setq num-fields (length line))
(setq fields-max (make-array num-fields :element-type 'fixnum
(incf count-lines))
(dotimes (i num-fields)
(setf (aref fields-av i) (float (/ (aref fields-av i) count-lines))))
- (list filename fields-max fields-av)))
+ (list (fil ufile) fields-max fields-av)))
;;; UMLS column/file functions
(defun make-ucol-for-column (colname filename ucols)
;; try to find column name without a terminal digit
(let* ((len (length colname))
- (last-digit? (digit-char-p (char colname (1- len))))
+ (last-digit? (digit-char-p (schar colname (1- len))))
(base-colname (if last-digit?
(subseq colname 0 (1- len))
colname))
(quote-str "'") (custom-value-fun))
(let ((ucol (make-instance
'ucol
- :col col :des des :ref ref :min min :av av :max max :fil fil
- :dty dty :sqltype sqltype :quote-str quote-str
+ :col col :des des :ref ref :min min :av av
+ :max (if (eql max 0) 1 max) ;; ensure at least one char wide
+ :fil fil
+ :dty dty
+ :sqltype sqltype
+ :quote-str quote-str
:parse-fun (ensure-compiled-fun parse-fun)
:custom-value-fun (ensure-compiled-fun custom-value-fun))))
(ensure-ucol-datatype ucol (datatype-for-colname col))
(defun find-ucols-for-ufile (ufile)
"Returns list of umls-cols for a file structure"
(loop for colname in (fields ufile)
- collect (find-ucol colname (fil ufile))))
+ collect (find-ucol colname
+ (if (subdir ufile)
+ (concatenate 'string (subdir ufile) "/" (fil ufile))
+ (fil ufile)))))
(defun umls-field-string-to-list (fmt)
"Converts a comma delimited list of fields into a list of field names. Will
(setf (gethash colname col-counts) 1)
colname))))))
-(defun make-ufile (fil des table cls rws bts fields)
- (let ((ufile (make-instance 'ufile :fil fil :des des :table table :cls cls
- :rws rws :bts bts :fields fields)))
- (setf (ucols ufile) (find-ucols-for-ufile ufile))
- ufile))
+(defun decompose-fil (fil)
+ (if fil
+ (let ((pos (position #\/ fil)))
+ (if pos
+ (values (subseq fil (1+ pos)) (subseq fil 0 pos))
+ (values fil nil)))
+ (values nil nil)))
+
+(defun filename-to-tablename (file)
+ (let ((pos (search ".RRF" file)))
+ (when pos
+ (setf file (subseq file 0 pos))))
+ (substitute #\_ #\. file))
+
+(defun make-ufile (dir fil des cls rws bts fields)
+ (multiple-value-bind (file subdir) (decompose-fil fil)
+ (let ((ufile (make-instance 'ufile :dir dir :fil file :subdir subdir
+ :des des :cls cls
+ :rws rws :bts bts :fields fields
+ :table (filename-to-tablename file))))
+ (setf (ucols ufile) (find-ucols-for-ufile ufile))
+ ufile)))
(defun datatype-for-colname (colname)
"Return datatype for column name"
(second (find colname +col-datatypes+ :key #'car :test #'string-equal)))
+(defun canonicalize-column-type (type)
+ (cond
+ ((string-equal type "SMALLINT")
+ (case *umls-sql-type*
+ (:mysql "SMALLINT")
+ ((:postgresql :postgresql-socket) "INT2")
+ (:oracle "NUMBER(5,0)")
+ (t "INTEGER")))
+ ((string-equal type "INTEGER")
+ (case *umls-sql-type*
+ (:mysql "INTEGER")
+ ((:postgresql :postgresql-socket) "INT4")
+ (:oracle "NUMBER(9,0)")
+ (t "INTEGER")))
+ ((string-equal type "BIGINT")
+ (case *umls-sql-type*
+ (:mysql "BIGINT")
+ ((:postgresql :postgresql-socket) "INT8")
+ (:oracle "NUMBER(38,0)")
+ (t "INTEGER")))
+ ((string-equal type "TEXT")
+ (case *umls-sql-type*
+ (:mysql "TEXT")
+ ((:postgresql :postgresql-socket) "TEXT")
+ (:oracle "VARCHAR2(3000)")
+ (t "VARCHAR(3000)")))
+ ((string-equal type "VARCHAR")
+ (case *umls-sql-type*
+ (:mysql "VARCHAR")
+ ((:postgresql :postgresql-socket) "VARCHAR")
+ (:oracle "VARCHAR2")
+ (t "VARCHAR")))
+ ((string-equal type "NUMERIC")
+ (case *umls-sql-type*
+ (:mysql "NUMERIC")
+ ((:postgresql :postgresql-socket) "NUMERIC")
+ (:oracle "NUMBER")
+ (t "NUMERIC")))
+ (t
+ type)))
+
(defun ensure-ucol-datatype (col datatype)
-"Add data type information to column"
+ "Add data type information to column"
(setf (datatype col) datatype)
(case datatype
- (sql-u (setf (sqltype col) "INTEGER"
+ (sql-u (setf (sqltype col) (canonicalize-column-type "INTEGER")
(parse-fun col) #'parse-ui
(quote-str col) ""))
- (sql-s (setf (sqltype col) "SMALLINT"
+ (sql-s (setf (sqltype col) (canonicalize-column-type "SMALLINT")
(parse-fun col) #'parse-integer
(quote-str col) ""))
- (sql-l (setf (sqltype col) "BIGINT"
+ (sql-l (setf (sqltype col) (canonicalize-column-type "BIGINT")
(parse-fun col) #'parse-integer
(quote-str col) ""))
- (sql-i (setf (sqltype col) "INTEGER"
+ (sql-i (setf (sqltype col) (canonicalize-column-type "INTEGER")
(parse-fun col) #'parse-integer
(quote-str col) ""))
- (sql-f (setf (sqltype col) "NUMERIC"
+ (sql-f (setf (sqltype col) (canonicalize-column-type "NUMERIC")
(parse-fun col) #'read-from-string
(quote-str col) ""))
- (t ; Default column type, optimized text storage
+ (t ; Default column type, optimized text storage
(setf (parse-fun col) #'add-sql-quotes
(quote-str col) "'")
(when (and (cmax col) (av col))
(if (> (cmax col) 255)
- (setf (sqltype col) "TEXT")
- (if (< (- (cmax col) (av col)) 4)
- (setf (sqltype col) "CHAR") ; if average bytes wasted < 4
- (setf (sqltype col) "VARCHAR")))))))
+ (setf (sqltype col) (canonicalize-column-type "TEXT"))
+ (setf (sqltype col) (canonicalize-column-type "VARCHAR")))))))
(defun escape-column-name (name)
(substitute #\_ #\/ name))
projects / umlisp.git / blobdiff