- (find-or-make-col-in-columns colname filename *umls-cols*))
-
-(defun find-umls-file (filename)
- "Returns umls-file structure for a filename"
- (find-if (lambda (f) (string-equal filename (umls-file-fil f))) *umls-files*))
-
-(defun umls-cols-for-umls-file (file)
- "Returns list of umls-cols for a file structure"
- (let ((filename (umls-file-fil file)))
- (mapcar (lambda (col) (find-umls-col col filename))
- (umls-file-fields file))))
-
-
-;;; Routines for analyzing cost of fixed size storage
-
-(defun umls-fixed-size-waste ()
- "Display storage waste if using all fixed size storage"
- (let ((totalwaste 0)
- (totalunavoidable 0)
- (totalavoidable 0)
- (unavoidable '())
- (avoidable '()))
- (dolist (file *umls-files*)
- (dolist (col (umls-file-colstructs file))
- (let* ((avwaste (- (umls-col-max col) (umls-col-av col)))
- (cwaste (* avwaste (umls-file-rws file))))
- (unless (zerop cwaste)
- (if (<= avwaste 6)
- (progn
- (incf totalunavoidable cwaste)
- (setq unavoidable (append unavoidable (list (list (umls-file-fil file) (umls-col-col col) avwaste cwaste)))))
- (progn
- (incf totalavoidable cwaste)
- (setq avoidable (append avoidable (list (list (umls-file-fil file) (umls-col-col col) avwaste cwaste))))))
- (incf totalwaste cwaste)))))
- (values totalwaste totalavoidable totalunavoidable avoidable unavoidable)))
-
-(defun display-waste ()
- (unless *umls-files*
- (init-umls))
- (multiple-value-bind (tw ta tu al ul) (umls-fixed-size-waste)
- (format t "Total waste: ~d~%" tw)
- (format t "Total avoidable: ~d~%" ta)
- (format t "Total unavoidable: ~d~%" tu)
- (format t "Avoidable:~%")
- (dolist (w al)
- (format t " (~a,~a): ~a,~a~%" (car w) (cadr w) (caddr w) (cadddr w)))
- (format t "Unavoidable:~%")
- (dolist (w ul)
- (format t " (~a,~a): ~a,~a~%" (car w) (cadr w) (caddr w) (cadddr w)))
- ))
-
-(defun max-umls-field ()
- "Return length of longest field"
- (unless *umls-files*
- (init-umls))
- (let ((max 0))
- (declare (fixnum max))
- (dolist (col *umls-cols*)
- (when (> (umls-col-max col) max)
- (setq max (umls-col-max col))))
- max))
-
-(defun max-umls-row ()
- "Return length of longest row"
- (if t
- 6000 ;;; hack to use on systems without MRCOLS/MRFILES -- ok for UMLS2001
- (progn
- (unless *umls-files*
- (init-umls))
- (let ((rowsizes '()))
- (dolist (file *umls-files*)
- (let ((row 0)
- (fields (umls-file-colstructs file)))
- (dolist (field fields)
- (incf row (1+ (umls-col-max field))))
- (push row rowsizes)))
- (car (sort rowsizes #'>))))))
+ (ensure-col-in-columns colname filename *umls-cols*))
+
+(defun find-ufile (filename)
+ "Returns umls-file structure for a filename"
+ (find-if #'(lambda (f) (string= filename (fil f))) *umls-files*))
+
+(defun position-field-file (filename fieldname)
+ "Returns the position of a field in a file"
+ (let ((ufile (find-ufile filename)))
+ (unless ufile
+ (warn "Unable to find ufile for filename ~A." filename)
+ (return-from position-field-file nil))
+ (let ((pos (position fieldname (fields ufile) :test #'string=)))
+ (unless pos
+ (warn "Unable to find field ~A in ufile ~S." fieldname ufile)
+ (return-from position-field-file nil))
+ pos)))
+
+(defun find-ucols-for-ufile (ufile)
+ "Returns list of umls-cols for a file structure"
+ (loop for colname in (fields ufile)
+ collect (find-ucol colname
+ (if (subdir ufile)
+ (concatenate 'string (subdir ufile) "/" (fil ufile))
+ (fil ufile)))))
+
+(defun umls-field-string-to-list (fmt)
+ "Converts a comma delimited list of fields into a list of field names. Will
+append a unique number (starting at 2) onto a column name that is repeated in the list"
+ (let ((col-counts (make-hash-table :test 'equal)))
+ (loop for colname in (delimited-string-to-list (escape-column-name fmt) #\,)
+ collect
+ (multiple-value-bind (value found) (gethash colname col-counts)
+ (cond
+ (found
+ (incf (gethash colname col-counts))
+ (concatenate 'string colname (write-to-string (1+ value))))
+ (t
+ (setf (gethash colname col-counts) 1)
+ colname))))))
+
+(defun decompose-fil (fil)
+ (if fil
+ (let ((pos (position #\/ fil)))
+ (if pos
+ (values (subseq fil (1+ pos)) (subseq fil 0 pos))
+ (values fil nil)))
+ (values nil nil)))
+
+(defun filename-to-tablename (file)
+ (let ((pos (search ".RRF" file)))
+ (when pos
+ (setf file (subseq file 0 pos))))
+ (substitute #\_ #\. file))
+
+(defun make-ufile (dir fil des cls rws bts fields)
+ (multiple-value-bind (file subdir) (decompose-fil fil)
+ (let ((ufile (make-instance 'ufile :dir dir :fil file :subdir subdir
+ :des des :cls cls
+ :rws rws :bts bts :fields fields
+ :table (filename-to-tablename file))))
+ ufile)))
+
+(defun set-ucols-for-ufiles (ufiles)
+ (dolist (ufile ufiles)
+ (setf (ucols ufile) (find-ucols-for-ufile ufile))))
+
+(defun datatype-for-colname (colname)
+"Return datatype for column name"
+ (second (find colname +col-datatypes+ :key #'car :test #'string-equal)))
+
+(defun canonicalize-column-type (type)
+ (cond
+ ((string-equal type "TINYINT")
+ (case *umls-sql-type*
+ (:mysql "TINYINT")
+ ((:postgresql :postgresql-socket) "INT1")
+ (:oracle "NUMBER(3,0)")
+ (t "INTEGER")))
+ ((string-equal type "SMALLINT")
+ (case *umls-sql-type*
+ (:mysql "SMALLINT")
+ ((:postgresql :postgresql-socket) "INT2")
+ (:oracle "NUMBER(5,0)")
+ (t "INTEGER")))
+ ((string-equal type "INTEGER")
+ (case *umls-sql-type*
+ (:mysql "INTEGER")
+ ((:postgresql :postgresql-socket) "INT4")
+ (:oracle "NUMBER(9,0)")
+ (t "INTEGER")))
+ ((string-equal type "BIGINT")
+ (case *umls-sql-type*
+ (:mysql "BIGINT")
+ ((:postgresql :postgresql-socket) "INT8")
+ (:oracle "NUMBER(38,0)")
+ (t "INTEGER")))
+ ((string-equal type "TEXT")
+ (case *umls-sql-type*
+ (:mysql "TEXT")
+ ((:postgresql :postgresql-socket) "TEXT")
+ (:oracle "VARCHAR2(3000)")
+ (t "VARCHAR(3000)")))
+ ((string-equal type "VARCHAR")
+ (case *umls-sql-type*
+ (:mysql "VARCHAR")
+ ((:postgresql :postgresql-socket) "VARCHAR")
+ (:oracle "VARCHAR2")
+ (t "VARCHAR")))
+ ((string-equal type "NUMERIC")
+ (case *umls-sql-type*
+ (:mysql "NUMERIC")
+ ((:postgresql :postgresql-socket) "NUMERIC")
+ (:oracle "NUMBER")
+ (t "NUMERIC")))
+ (t
+ type)))
+
+(defun ensure-ucol-datatype (col datatype)
+ "Add data type information to column"
+ (setf (datatype col) datatype)
+ (case datatype
+ (sql-u (setf (sqltype col) (canonicalize-column-type "INTEGER")
+ (parse-fun col) #'parse-ui
+ (quote-str col) ""))
+ (sql-s (setf (sqltype col) (canonicalize-column-type "SMALLINT")
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
+ (sql-l (setf (sqltype col) (canonicalize-column-type "BIGINT")
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
+ (sql-i (setf (sqltype col) (canonicalize-column-type "INTEGER")
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
+ (sql-t (setf (sqltype col) (canonicalize-column-type "TINYINT")
+ (parse-fun col) #'parse-integer
+ (quote-str col) ""))
+ (sql-f (setf (sqltype col) (canonicalize-column-type "NUMERIC")
+ (parse-fun col) #'read-from-string
+ (quote-str col) ""))
+ (t ; Default column type, optimized text storage
+ (setf (parse-fun col) #'add-sql-quotes
+ (quote-str col) "'")
+ (when (and (cmax col) (av col))
+ (if (> (cmax col) 255)
+ (setf (sqltype col) (canonicalize-column-type "TEXT"))
+ (setf (sqltype col) (canonicalize-column-type "VARCHAR")))))))
+
+(defun escape-column-name (name)
+ (substitute #\_ #\/ name))