X-Git-Url: http://git.kpe.io/?p=umlisp.git;a=blobdiff_plain;f=parse-common.lisp;h=b1395665b68eb612ef4de7fcfccaa16436ce2981;hp=4c13a526f45f5a179ac102bc0c4de0f753843291;hb=HEAD;hpb=bfdd5c9d3d66970759fcdbee5a51da2ca93ddf06 diff --git a/parse-common.lisp b/parse-common.lisp index 4c13a52..f7fab1e 100644 --- a/parse-common.lisp +++ b/parse-common.lisp @@ -2,469 +2,393 @@ ;;;; ************************************************************************* ;;;; FILE IDENTIFICATION ;;;; -;;;; Name: parse-common.lisp -;;;; Purpose: Common, stable parsing routines for UMLisp -;;;; Programmer: Kevin M. Rosenberg -;;;; Date Started: Apr 2000 -;;;; -;;;; $Id: parse-common.lisp,v 1.2 2002/10/09 23:03:41 kevin Exp $ +;;;; Name: parse-common.lisp +;;;; Purpose: Common, stable parsing routines for UMLisp +;;;; Author: Kevin M. Rosenberg +;;;; Created: Apr 2000 ;;;; ;;;; This file, part of UMLisp, is -;;;; Copyright (c) 2000-2002 by Kevin M. Rosenberg, M.D. +;;;; Copyright (c) 2000-2010 by Kevin M. Rosenberg, M.D. ;;;; ;;;; UMLisp users are granted the rights to distribute and use this software ;;;; as governed by the terms of the GNU General Public License. ;;;; ************************************************************************* -(in-package :umlisp) -(declaim (optimize (speed 3) (safety 1) (compilation-speed 0) (debug 3))) +(in-package #:umlisp) + +(defun ensure-ucols+ufiles (&optional (alwaysclear nil)) + "Initialize all UMLS file and column structures if not already initialized" + (handler-case + (when (or alwaysclear (null *umls-files*)) + (setf *umls-cols* nil *umls-files* nil) + (gen-ufiles) + (gen-ucols) + (set-ucols-for-ufiles *umls-files*) + (ensure-field-lengths)) + (error (e) + (warn "Error reading ucols+ufiles: ~A." e) + (setf *umls-cols* nil *umls-files* nil) + nil)) + t) + + +(defun add-ucols (ucols) + "Adds a ucol or list of ucols to *umls-cols*. Returns input value." + (setq *umls-cols* (append (mklist ucols) *umls-cols*)) + ucols) + +(defun add-ufiles (ufiles) + "Adds a ufile or list of ufiles to *umls-filess*. Returns input value." + (setq *umls-files* (append (mklist ufiles) *umls-files*)) + ufiles) + +(defun ufile-pathname (ufile &optional (extension "")) + "Return pathname for a umls filename with an optional extension" + (assert (typep ufile 'ufile)) + (let* ((dirs (nconc (list (dir ufile)) + (awhen (subdir ufile) (list it)))) + (name-list (delimited-string-to-list (fil ufile) #\.)) + (name (if (second name-list) + (first name-list) + (concatenate 'string (first name-list) (or extension "")))) + (type (when (second name-list) + (concatenate 'string (second name-list) (or extension ""))))) + (merge-pathnames + (make-pathname :name name :type type + :directory (cons :relative dirs)) + *umls-path*))) (defun umls-pathname (filename &optional (extension "")) -"Return pathname for a umls filename" +"Return pathname for a umls filename with an optional extension" (etypecase filename (string - (merge-pathnames - (make-pathname :name (concatenate 'string filename extension)) - (case (char filename 0) - ((#\M #\m) - *meta-path*) - ((#\L #\l) - *lex-path*) - ((#\S #\s) - *net-path*) - (t - *umls-path*)))) + (let* ((name-list (delimited-string-to-list filename #\.)) + (name (if (second name-list) + (first name-list) + (concatenate 'string (first name-list) (or extension "")))) + (type (when (second name-list) + (concatenate 'string (second name-list) (or extension ""))))) + (merge-pathnames + (make-pathname :name name :type type) + (case (schar filename 0) + ((#\M #\m) + *meta-path*) + ((#\L #\l) + *lex-path*) + ((#\S #\s) + *net-path*) + (t + *umls-path*))))) (pathname - filename))) - -(defun read-umls-line (strm) - "Read a line from a UMLS stream, split into fields" - (let ((line (read-line strm nil 'eof))) - (if (stringp line) ;; ensure not 'eof - (let* ((len (length line)) - (maybe-remove-terminal ;; LRWD doesn't have '|' at end of line - (if (char= #\| (char line (1- len))) - (subseq line 0 (1- len)) - line))) - (declare (fixnum len)) - (delimited-string-to-list maybe-remove-terminal #\|)) - line))) + filename))) ;;; Find field lengths for LEX and NET files -(defun file-field-lengths (files) - (let ((lengths '())) - (dolist (file files) - (setq file (umls-file-fil file)) - (let (max-field count-field num-fields (count-lines 0)) - (with-umls-file (fields file) - (unless num-fields - (setq num-fields (length fields)) - (setq max-field (make-array num-fields :element-type 'fixnum - :initial-element 0)) - (setq count-field (make-array num-fields :element-type 'number - :initial-element 0))) - (dotimes (i (length fields)) - (declare (fixnum i)) - (let ((len (length (nth i fields)))) - (incf (aref count-field i) len) - (when (> len (aref max-field i)) - (setf (aref max-field i) len)))) - (incf count-lines)) - (dotimes (i num-fields) - (setf (aref count-field i) (float (/ (aref count-field i) count-lines)))) - (push (list file max-field count-field) lengths))) - (nreverse lengths))) - -(defun init-field-lengths () +(defun ensure-field-lengths () "Initial colstruct field lengths for files that don't have a measurement. Currently, these are the LEX and NET files." - (let ((measure-files '())) - (dolist (file *umls-files*) - (let ((filename (umls-file-fil file))) - (unless (or (char= #\M (char filename 0)) - (char= #\m (char filename 0))) - (push file measure-files)))) - (let ((length-lists (file-field-lengths measure-files))) - (dolist (length-list length-lists) - (let* ((filename (car length-list)) - (max-field (cadr length-list)) - (av-field (caddr length-list)) - (file (find-umls-file filename))) - (when file - (if (/= (length max-field) (length (umls-file-fields file))) - (format t "Warning: Number of file fields ~A doesn't match length of fields in file structure ~S" - max-field file) - (dotimes (i (max (length max-field) (length (umls-file-fields file)))) - (declare (fixnum i)) - (let* ((field (nth i (umls-file-fields file))) - (col (find-umls-col field filename))) - (if col - (progn - (setf (umls-col-max col) (aref max-field i)) - (setf (umls-col-av col) (aref av-field i)) - (add-datatype-to-col col (datatype-for-col (umls-col-col col)))) - (error "can't find column ~A" field))))))))))) - - + (dolist (length-list (ufiles-field-lengths (ufiles-to-measure))) + (destructuring-bind (filename fields-max fields-av count-lines) length-list + (let ((file (find-ufile filename))) + (unless file + (error "Can't find ~A filename in ufiles" filename)) + (if (zerop count-lines) + (warn "File ~A is empty." filename) + (progn + (unless (= (length fields-max) (length (fields file))) + (error + "Number of file fields ~A not equal to field count in ufile ~S" + fields-max file)) + (dotimes (i (length (fields file))) + (declare (fixnum i)) + (let* ((field (nth i (fields file))) + (col (find-ucol field filename))) + (unless col + (error "can't find column ~A" field)) + (setf (cmax col) (aref fields-max i)) + (setf (av col) (aref fields-av i)) + (ensure-ucol-datatype col (datatype-for-colname (col col))))))))))) + +(defun ufiles-to-measure () + "Returns a list of ufiles to measure" + (loop for ufile in *umls-files* + unless (or (char= #\M (schar (fil ufile) 0)) + (char= #\m (schar (fil ufile) 0))) + collect ufile)) + + +(defun ufiles-field-lengths (ufiles) + "Returns a list of lists of containing (FILE MAX AV)" + (loop for ufile in ufiles collect (file-field-lengths ufile))) + +(defun file-field-lengths (ufile) + "Returns a list of FILENAME MAX AV" + (declare (optimize (speed 3) (safety 0))) + (let (fields-max fields-av num-fields (count-lines 0)) + (declare (fixnum count-lines)) + (with-umls-ufile (line ufile) + (unless num-fields + (setq num-fields (length line)) + (setq fields-max (make-array num-fields :element-type 'fixnum + :initial-element 0)) + (setq fields-av (make-array num-fields :element-type '(or integer float) + :initial-element 0))) + (dotimes (i (or num-fields 0)) + (declare (fixnum i)) + (let* ((str (nth i line)) + (len (length #-(and clisp unicode) str + #+(and clisp unicode) + (if *octet-sql-storage* + (ext:convert-string-to-bytes str charset:utf-8) + str)))) + #-(and clisp unicode) (declare (string str)) + (declare (type (integer 0 100000000) len)) + (incf (aref fields-av i) len) + (when (> len (aref fields-max i)) + (setf (aref fields-max i) len)))) + (incf count-lines)) + (dotimes (i (or num-fields 0)) + (setf (aref fields-av i) + (if (plusp count-lines) + (float (/ (aref fields-av i) count-lines)) + 0))) + (list (fil ufile) fields-max fields-av count-lines))) ;;; UMLS column/file functions -(defun find-col-in-columns (colname filename cols) +(defun find-ucol-of-colname (colname filename ucols) "Returns list of umls-col structure for a column name and a filename" - (dolist (col cols) - (when (and (string-equal filename (umls-col-fil col)) - (string-equal colname (umls-col-col col))) - (return-from find-col-in-columns col))) - nil) - -(defun find-or-make-col-in-columns (colname filename cols) - (let ((col (find-col-in-columns colname filename cols))) - (if col - col - ;; try to find column name without a terminal digit - (let* ((last-char (char colname (1- (length colname)))) - (digit (- (char-code last-char) (char-code #\0)))) - (if (and (>= digit 0) (<= digit 9)) - (let ((base-colname (subseq colname 0 (1- (length colname))))) - (setq col (find-col-in-columns base-colname filename cols)) - (if col - (let ((new-col (make-umls-col - :col (copy-seq colname) - :des (copy-seq (umls-col-des col)) - :ref (copy-seq (umls-col-ref col)) - :min (umls-col-min col) - :max (umls-col-max col) - :fil (copy-seq (umls-col-fil col)) - :sqltype (copy-seq (umls-col-sqltype col)) - :dty (copy-seq (umls-col-dty col)) - :parsefunc (umls-col-parsefunc col) - :quotechar (copy-seq (umls-col-quotechar col)) - :datatype (umls-col-datatype col) - :custom-value-func (umls-col-custom-value-func col)))) - (push new-col *umls-cols*) - new-col) - (error "Couldn't find a base column for col ~A in file ~A" - colname filename))) - (let ((new-col (make-umls-col - :col (copy-seq colname) - :des "Unknown" - :ref "" - :min nil - :max nil - :fil filename - :sqltype "VARCHAR" - :dty nil - :parsefunc #'add-sql-quotes - :quotechar "'" - :datatype nil - :custom-value-func nil))) - (push new-col *umls-cols*) - new-col)))))) - -(defun find-umls-col (colname filename) + (dolist (ucol ucols nil) + (when (and (string-equal filename (fil ucol)) + (string-equal colname (col ucol))) + (return-from find-ucol-of-colname ucol)))) + +(defun ensure-col-in-columns (colname filename ucols) + (aif (find-ucol-of-colname colname filename ucols) + it + (add-ucols (make-ucol-for-column colname filename ucols)))) + +(defun make-ucol-for-column (colname filename ucols) + ;; try to find column name without a terminal digit + (let* ((len (length colname)) + (last-digit? (digit-char-p (schar colname (1- len)))) + (base-colname (if last-digit? + (subseq colname 0 (1- len)) + colname)) + (ucol (when last-digit? + (find-ucol-of-colname base-colname filename ucols)))) + (when (and last-digit? (null ucol)) + (error "Couldn't find a base column for col ~A in file ~A" + colname filename)) + (copy-or-new-ucol colname filename ucol))) + +(defun copy-or-new-ucol (colname filename ucol) + (if ucol + (make-instance + 'ucol + :col (copy-seq colname) :des (copy-seq (des ucol)) :ref (copy-seq (ref ucol)) + :min (cmin ucol) :max (cmax ucol) :fil (copy-seq (fil ucol)) + :sqltype (copy-seq (sqltype ucol)) :dty (copy-seq (dty ucol)) + :parse-fun (parse-fun ucol) :quote-str (copy-seq (quote-str ucol)) + :datatype (datatype ucol) :custom-value-fun (custom-value-fun ucol)) + (make-empty-ucol colname filename))) + +(defun ensure-compiled-fun (fun) + "Ensure that a function is compiled" + (etypecase fun + (null + nil) + (function + (if (compiled-function-p fun) + fun + (compile nil fun))) + (list + (compile nil fun)))) + +(defun make-ucol (col des ref min av max fil dty + &key (sqltype "VARCHAR") (parse-fun #'add-sql-quotes) + (quote-str "'") (custom-value-fun)) + (let ((ucol (make-instance + 'ucol + :col col :des des :ref ref :min min :av av + :max (if (eql max 0) 1 max) ;; ensure at least one char wide + :fil fil + :dty dty + :sqltype sqltype + :quote-str quote-str + :parse-fun (ensure-compiled-fun parse-fun) + :custom-value-fun (ensure-compiled-fun custom-value-fun)))) + (ensure-ucol-datatype ucol (datatype-for-colname col)) + ucol)) + +(defun make-empty-ucol (colname filename) + ;;(format "call in make-empty-ucol: ~A/~A" colname filename) + (make-ucol (copy-seq colname) "Unknown" "" nil nil nil filename nil)) + +(defun find-ucol (colname filename) "Returns list of umls-col structure for a column name and a filename" - (find-or-make-col-in-columns colname filename *umls-cols*)) - -(defun find-umls-file (filename) - "Returns umls-file structure for a filename" - (find-if (lambda (f) (string-equal filename (umls-file-fil f))) *umls-files*)) - -(defun umls-cols-for-umls-file (file) - "Returns list of umls-cols for a file structure" - (let ((filename (umls-file-fil file))) - (mapcar (lambda (col) (find-umls-col col filename)) - (umls-file-fields file)))) - - -;; SQL command functions - -(defun create-table-cmd (file) -"Return sql command to create a table" - (let ((col-func - (lambda (c) - (let ((sqltype (umls-col-sqltype c))) - (concatenate 'string (umls-col-col c) - " " - (if (or (string-equal sqltype "VARCHAR") - (string-equal sqltype "CHAR")) - (format nil "~a (~a)" sqltype (umls-col-max c)) - sqltype) - ","))))) - (format nil "CREATE TABLE ~a (~a)" (umls-file-table file) - (string-trim-last-character - (mapcar-append-string col-func (umls-cols-for-umls-file file)))))) - -(defun create-custom-table-cmd (tablename sql-cmd) -"Return SQL command to create a custom table" - (format nil "CREATE TABLE ~a AS ~a;" tablename sql-cmd)) - -(defun insert-values-cmd (file values) -"Return sql insert command for a row of values" - (let ((insert-func - (lambda (col value) - (concatenate - 'string - (umls-col-quotechar col) - (if (null (umls-col-parsefunc col)) - value - (format nil "~A" (funcall (umls-col-parsefunc col) value))) - (umls-col-quotechar col) - ",")))) - (format - nil "INSERT INTO ~a (~a) VALUES (~a)" - (umls-file-table file) - (string-trim-last-character - (mapcar-append-string (lambda (c) (concatenate 'string c ",")) - (umls-file-fields file))) - (string-trim-last-character - (concatenate 'string - (mapcar2-append-string insert-func - (remove-custom-cols (umls-file-colstructs file)) - values) - (custom-col-values (custom-colstructs-for-file file) values "," t))) - ))) - -(defun custom-col-values (colstructs values delim doquote) - "Returns string of column values for SQL inserts for custom columns" - (let ((result "")) - (dolist (col colstructs) - (let* ((func (umls-col-custom-value-func col)) - (custom-value (funcall func values))) - (string-append result - (if doquote (umls-col-quotechar col)) - (escape-backslashes custom-value) - (if doquote (umls-col-quotechar col)) - delim))) - result)) - -(defun remove-custom-cols (cols) - "Remove custom cols from a list col umls-cols" - (remove-if #'umls-col-custom-value-func cols)) - -(defun find-custom-cols-for-filename (filename) - (remove-if-not (lambda (x) (string-equal filename (car x))) +custom-cols+)) - -(defun find-custom-col (filename col) - (find-if (lambda (x) (and (string-equal filename (car x)) - (string-equal col (cadr x)))) +custom-cols+)) - - -(defun custom-colnames-for-filename (filename) - (mapcar #'cadr (find-custom-cols-for-filename filename))) - -(defun custom-colstructs-for-file (file) - (remove-if-not #'umls-col-custom-value-func (umls-file-colstructs file))) - -(defun noneng-lang-index-files () - (remove-if-not (lambda (f) (and (> (length (umls-file-fil f)) 4) - (string-equal (umls-file-fil f) "MRXW." :end1 5) - (not (string-equal (umls-file-fil f) "MRXW.ENG")) - (not (string-equal (umls-file-fil f) "MRXW.NONENG")))) - *umls-files*)) - -;;; SQL Command Functions - -(defun create-index-cmd (colname tablename length) -"Return sql create index command" - (format nil "CREATE INDEX ~a ON ~a (~a ~a)" - (concatenate 'string tablename "_" colname "_X") tablename colname - (if (integerp length) - (format nil "(~d)" length) - ""))) - -(defun create-all-tables-cmdfile () -"Return sql commands to create all tables. Not need for automated SQL import" - (mapcar (lambda (f) (format nil "~a~%~%" (create-table-cmd f))) *umls-files*)) - - -;; SQL Execution functions - -(defun sql-drop-tables (conn) -"SQL Databases: drop all tables" - (mapcar - (lambda (file) - (ignore-errors - (sql-execute (format nil "DROP TABLE ~a" (umls-file-table file)) conn))) - *umls-files*)) - -(defun sql-create-tables (conn) -"SQL Databases: create all tables" - (mapcar (lambda (file) (sql-execute (create-table-cmd file) conn)) *umls-files*)) - -(defun sql-create-custom-tables (conn) -"SQL Databases: create all custom tables" - (mapcar (lambda (ct) - (sql-execute (create-custom-table-cmd (car ct) (cadr ct)) conn)) - +custom-tables+)) - -(defun sql-insert-values (conn file) -"SQL Databases: inserts all values for a file" - (with-umls-file (line (umls-file-fil file)) - (sql-execute (insert-values-cmd file line) conn))) - -(defun sql-insert-all-values (conn) -"SQL Databases: inserts all values for all files" - (mapcar (lambda (file) (sql-insert-values conn file)) *umls-files*)) - -(defun sql-create-indexes (conn &optional (indexes +index-cols+)) -"SQL Databases: create all indexes" -(mapcar - (lambda (idx) - (sql-execute (create-index-cmd (car idx) (cadr idx) (caddr idx)) conn)) - indexes)) - -(defun create-umls-db-by-insert () -"SQL Databases: initializes entire database via SQL insert commands" - (init-umls) - (init-hash-table) - (with-sql-connection (conn) -;; (sql-drop-tables conn) -;; (sql-create-tables conn) -;; (sql-insert-all-values conn) - (sql-create-indexes conn) - (sql-create-custom-tables conn) - (sql-create-indexes conn +custom-index-cols+))) - -(defun create-umls-db (&optional (extension ".trans") - (copy-cmd #'mysql-copy-cmd)) - "SQL Databases: initializes entire database via SQL copy commands" - (init-umls) - (init-hash-table) - (translate-all-files extension) - (with-sql-connection (conn) - (sql-drop-tables conn) - (sql-create-tables conn) - (mapcar - #'(lambda (file) (sql-execute (funcall copy-cmd file extension) conn)) - *umls-files*) - (sql-create-indexes conn) - (sql-create-custom-tables conn) - (sql-create-indexes conn +custom-index-cols+))) - -(defun translate-all-files (&optional (extension ".trans")) -"Copy translated files and return postgresql copy commands to import" - (make-noneng-index-file extension) - (mapcar (lambda (f) (translate-file f extension)) *umls-files*)) - -(defun translate-file (file extension) - "Translate a umls file into a format suitable for sql copy cmd" - (let ((path (umls-pathname (umls-file-fil file) extension))) - (if (probe-file path) - (progn - (format t "File ~A already exists: skipping~%" path) - nil) - (with-open-file (ostream path :direction :output) - (with-umls-file (line (umls-file-fil file)) - (princ (umls-translate file line) ostream) - (princ #\newline ostream)) - t)))) - -(defun make-noneng-index-file (extension) - "Make non-english index file" - (let* ((outfile (find-umls-file "MRXW.NONENG")) - (path (umls-pathname (umls-file-fil outfile) extension))) - - (if (probe-file path) - (progn - (format t "File ~A already exists: skipping~%" path) - nil) - (progn - (with-open-file (ostream path :direction :output) - (dolist (inputfile (noneng-lang-index-files)) - (with-umls-file (line (umls-file-fil inputfile)) - (princ (umls-translate outfile line) ostream) ;; use outfile for custom cols - (princ #\newline ostream)))) - t)))) - -(defun pg-copy-cmd (file extension) -"Return postgresql copy statement for a file" - (format nil "COPY ~a FROM '~a' using delimiters '|' with null as ''" - (umls-file-table file) (umls-pathname (umls-file-fil file) extension))) - -(defun mysql-copy-cmd (file extension) -"Return mysql copy statement for a file" - (format nil "LOAD DATA LOCAL INFILE \"~a\" INTO TABLE ~a FIELDS TERMINATED BY \"|\"" - (umls-pathname (umls-file-fil file) extension) (umls-file-table file))) - -(defun umls-translate (file line) -"Translate a single line for sql output" -(string-trim-last-character - (concatenate 'string - (mapcar2-append-string - (lambda (col value) - (concatenate - 'string - (if (eq (umls-col-datatype col) 'sql-u) - (format nil "~d" (parse-ui value "")) - (escape-backslashes value)) - "|")) - (remove-custom-cols (umls-file-colstructs file)) - line) - (custom-col-values (custom-colstructs-for-file file) line "|" nil)))) - - -(defun umls-fixed-size-waste () - "Display storage waste if using all fixed size storage" - (let ((totalwaste 0) - (totalunavoidable 0) - (totalavoidable 0) - (unavoidable '()) - (avoidable '())) - (dolist (file *umls-files*) - (dolist (col (umls-file-colstructs file)) - (let* ((avwaste (- (umls-col-max col) (umls-col-av col))) - (cwaste (* avwaste (umls-file-rws file)))) - (unless (zerop cwaste) - (if (<= avwaste 6) - (progn - (incf totalunavoidable cwaste) - (setq unavoidable (append unavoidable (list (list (umls-file-fil file) (umls-col-col col) avwaste cwaste))))) - (progn - (incf totalavoidable cwaste) - (setq avoidable (append avoidable (list (list (umls-file-fil file) (umls-col-col col) avwaste cwaste)))))) - (incf totalwaste cwaste))))) - (values totalwaste totalavoidable totalunavoidable avoidable unavoidable))) - -(defun display-waste () - (unless *umls-files* - (init-umls)) - (multiple-value-bind (tw ta tu al ul) (umls-fixed-size-waste) - (format t "Total waste: ~d~%" tw) - (format t "Total avoidable: ~d~%" ta) - (format t "Total unavoidable: ~d~%" tu) - (format t "Avoidable:~%") - (dolist (w al) - (format t " (~a,~a): ~a,~a~%" (car w) (cadr w) (caddr w) (cadddr w))) - (format t "Unavoidable:~%") - (dolist (w ul) - (format t " (~a,~a): ~a,~a~%" (car w) (cadr w) (caddr w) (cadddr w))) - )) - -(defun max-umls-field () - "Return length of longest field" - (unless *umls-files* - (init-umls)) - (let ((max 0)) - (declare (fixnum max)) - (dolist (col *umls-cols*) - (when (> (umls-col-max col) max) - (setq max (umls-col-max col)))) - max)) - -(defun max-umls-row () - "Return length of longest row" - (if t - 6000 ;;; hack to use on systems without MRCOLS/MRFILES -- ok for UMLS2001 - (progn - (unless *umls-files* - (init-umls)) - (let ((rowsizes '())) - (dolist (file *umls-files*) - (let ((row 0) - (fields (umls-file-colstructs file))) - (dolist (field fields) - (incf row (1+ (umls-col-max field)))) - (push row rowsizes))) - (car (sort rowsizes #'>)))))) + (ensure-col-in-columns colname filename *umls-cols*)) + +(defun find-ufile (filename) + "Returns umls-file structure for a filename" + (find-if #'(lambda (f) (string= filename (fil f))) *umls-files*)) + +(defun position-field-file (filename fieldname) + "Returns the position of a field in a file" + (let ((ufile (find-ufile filename))) + (unless ufile + (warn "Unable to find ufile for filename ~A." filename) + (return-from position-field-file nil)) + (let ((pos (position fieldname (fields ufile) :test #'string=))) + (unless pos + (warn "Unable to find field ~A in ufile ~S." fieldname ufile) + (return-from position-field-file nil)) + pos))) + +(defun find-ucols-for-ufile (ufile) + "Returns list of umls-cols for a file structure" + (loop for colname in (fields ufile) + collect (find-ucol colname + (if (subdir ufile) + (concatenate 'string (subdir ufile) "/" (fil ufile)) + (fil ufile))))) + +(defun umls-field-string-to-list (fmt) + "Converts a comma delimited list of fields into a list of field names. Will +append a unique number (starting at 2) onto a column name that is repeated in the list" + (let ((col-counts (make-hash-table :test 'equal))) + (loop for colname in (delimited-string-to-list (escape-column-name fmt) #\,) + collect + (multiple-value-bind (value found) (gethash colname col-counts) + (cond + (found + (incf (gethash colname col-counts)) + (concatenate 'string colname (write-to-string (1+ value)))) + (t + (setf (gethash colname col-counts) 1) + colname)))))) + +(defun decompose-fil (fil) + (if fil + (let ((pos (position #\/ fil))) + (if pos + (values (subseq fil (1+ pos)) (subseq fil 0 pos)) + (values fil nil))) + (values nil nil))) + +(defun filename-to-tablename (file) + (let ((pos (search ".RRF" file))) + (when pos + (setf file (subseq file 0 pos)))) + (substitute #\_ #\. file)) + +(defun make-ufile (dir fil des cls rws bts fields) + (multiple-value-bind (file subdir) (decompose-fil fil) + (let ((ufile (make-instance 'ufile :dir dir :fil file :subdir subdir + :des des :cls cls + :rws rws :bts bts :fields fields + :table (filename-to-tablename file)))) + ufile))) + +(defun set-ucols-for-ufiles (ufiles) + (dolist (ufile ufiles) + (setf (ucols ufile) (find-ucols-for-ufile ufile)))) + +(defun datatype-for-colname (colname) +"Return datatype for column name" + (second (find colname +col-datatypes+ :key #'car :test #'string-equal))) + +(defun canonicalize-column-type (type) + (cond + ((string-equal type "TINYINT") + (case *umls-sql-type* + (:mysql "TINYINT") + ((:postgresql :postgresql-socket) "INT1") + (:oracle "NUMBER(3,0)") + (t "INTEGER"))) + ((string-equal type "SMALLINT") + (case *umls-sql-type* + (:mysql "SMALLINT") + ((:postgresql :postgresql-socket) "INT2") + (:oracle "NUMBER(5,0)") + (t "INTEGER"))) + ((string-equal type "INTEGER") + (case *umls-sql-type* + (:mysql "INTEGER") + ((:postgresql :postgresql-socket) "INT4") + (:oracle "NUMBER(9,0)") + (t "INTEGER"))) + ((string-equal type "BIGINT") + (case *umls-sql-type* + (:mysql "BIGINT") + ((:postgresql :postgresql-socket) "INT8") + (:oracle "NUMBER(38,0)") + (t "INTEGER"))) + ((string-equal type "TEXT") + (case *umls-sql-type* + (:mysql "TEXT") + ((:postgresql :postgresql-socket) "TEXT") + (:oracle "VARCHAR2(3000)") + (t "VARCHAR(3000)"))) + ((string-equal type "VARCHAR") + (case *umls-sql-type* + (:mysql "VARCHAR") + ((:postgresql :postgresql-socket) "VARCHAR") + (:oracle "VARCHAR2") + (t "VARCHAR"))) + ((string-equal type "NUMERIC") + (case *umls-sql-type* + (:mysql "NUMERIC") + ((:postgresql :postgresql-socket) "NUMERIC") + (:oracle "NUMBER") + (t "NUMERIC"))) + (t + type))) + +(defun ensure-ucol-datatype (col datatype) + "Add data type information to column" + (setf (datatype col) datatype) + (case datatype + (sql-u (setf (sqltype col) (canonicalize-column-type "INTEGER") + (parse-fun col) #'parse-ui + (quote-str col) "")) + (sql-s (setf (sqltype col) (canonicalize-column-type "SMALLINT") + (parse-fun col) #'parse-integer + (quote-str col) "")) + (sql-l (setf (sqltype col) (canonicalize-column-type "BIGINT") + (parse-fun col) #'parse-integer + (quote-str col) "")) + (sql-i (setf (sqltype col) (canonicalize-column-type "INTEGER") + (parse-fun col) #'parse-integer + (quote-str col) "")) + (sql-t (setf (sqltype col) (canonicalize-column-type "TINYINT") + (parse-fun col) #'parse-integer + (quote-str col) "")) + (sql-f (setf (sqltype col) (canonicalize-column-type "NUMERIC") + (parse-fun col) #'read-from-string + (quote-str col) "")) + (t ; Default column type, optimized text storage + (setf (parse-fun col) #'add-sql-quotes + (quote-str col) "'") + (when (and (cmax col) (av col)) + (if (> (cmax col) 255) + (setf (sqltype col) (canonicalize-column-type "TEXT")) + (setf (sqltype col) (canonicalize-column-type "VARCHAR"))))))) + +(defun escape-column-name (name) + (substitute #\_ #\/ name)) + +;; SQLNAME is required for collision of SQL reserved words (MYSQL 8: RANK) +;; and column names in UMLS (RANK in MRRANK) +(defvar *sql-reserved-names* '("RANK")) +(defmethod sqlname ((c ucol)) + (sqlname (col c))) +(defmethod sqlname ((name string)) + (if (find name *sql-reserved-names* :test #'string-equal) + (concatenate 'string "_" name) + name)) +(defmethod sqlname ((l list)) + (mapcar #'sqlname l)) +(defmethod sqlname ((s symbol)) + (sqlname (symbol-name s)))