" MAX_ROWS=200000000"
"")
(if (eq *umls-sql-type* :mysql)
- " TYPE=MYISAM DEFAULT CHARACTER latin1"
+ " TYPE=MYISAM CHARACTER SET utf8"
""))))
(defun create-custom-table-cmd (tablename sql-cmd)
(format nil " (~d)" length)
"")))
((:postgresql :postgresql-socket)
- ;; FIXME: incorrect syntax
+ ;; FIXME: incorrect syntax for postgresql?
(if (integerp length)
(format nil "substr((~A)::text,1,~D)" colname length)
colname))
(:oracle "NUMBER(2,0)")
(t "INTEGER")))
:database conn)
+ ;; KCON deprecated by KPFENG field in MRCONSO
+ #+nil
(dolist (tuple (query "select distinct cui from MRCONSO order by cui"
:database conn))
(let ((cui (car tuple)))
(format nil "DROP INDEX ~a"
(concatenate 'string tablename "_" colname "_X")))))
-(defun sql-create-indexes (conn &optional (indexes +index-cols+))
+(defun sql-create-indexes (conn &key (indexes +index-cols+) verbose)
"SQL Databases: create all indexes"
(dolist (idx indexes)
+ (when verbose (format t "UMLS Import: Creating index for column ~A on table ~A.~%"
+ (first idx) (second idx)))
(ignore-errors (sql-execute (drop-index-cmd (car idx) (cadr idx)) conn))
(sql-execute (create-index-cmd (car idx) (cadr idx) (caddr idx)) conn)))
(make-usrl conn)
(make-ustats))
-(defun create-umls-db-by-insert ()
+(defun create-umls-db-by-insert (&key verbose)
"SQL Databases: initializes entire database via SQL insert commands"
(ensure-ucols+ufiles)
(ensure-preparse)
(sql-insert-all-values conn)
(sql-create-indexes conn)
(sql-create-custom-tables conn)
- (sql-create-indexes conn +custom-index-cols+)
+ (sql-create-indexes conn :indexes +custom-index-cols+ :verbose verbose)
(sql-create-special-tables conn)))
-(defun create-umls-db (&key (extension ".trans") (skip-translation nil))
+(defun create-umls-db (&key (extension "-trans") (force-translation nil) (verbose nil))
"SQL Databases: initializes entire database via SQL copy commands.
This is much faster that using create-umls-db-insert."
+ (when verbose (format t "UMLS Import: Starting.~%"))
(ensure-ucols+ufiles)
+ (when verbose (format t "UMLS Import: Preparsing files.~%"))
(ensure-preparse)
- (unless skip-translation
- (translate-all-files extension))
+ (when verbose (format t "UMLS Import: Converting text UMLS files to optimized format.~%"))
+ (translate-all-files :extension extension :verbose verbose :force force-translation)
(let ((copy-cmd
(ecase (umls-sql-type)
(:mysql #'mysql-copy-cmd)
(sql-drop-tables conn)
(sql-create-tables conn)
(dolist (file *umls-files*)
+ (when verbose (format t "UMLS Import: Importing file ~A to SQL.~%" (fil file)))
(sql-execute (funcall copy-cmd file extension) conn))
- (sql-create-indexes conn)
+ (When verbose (format t "UMLS Import: Creating SQL indices.~%"))
+ (sql-create-indexes conn :verbose verbose)
+ (When verbose (format t "UMLS Import: Creating custom tables.~%"))
(sql-create-custom-tables conn)
- (sql-create-indexes conn +custom-index-cols+)
- (sql-create-special-tables conn))))
-
-(defun translate-all-files (&optional (extension ".trans"))
- "Copy translated files and return postgresql copy commands to import"
- (make-noneng-index-file extension)
+ (When verbose (format t "UMLS Import: Creating custom indices.~%"))
+ (sql-create-indexes conn :indexes +custom-index-cols+ :verbose verbose)
+ (When verbose (format t "UMLS Import: Creating special tables.~%"))
+ (sql-create-special-tables conn)))
+ (When verbose (format t "UMLS Import: Completed.~%"))
+ t)
+
+(defun translate-all-files (&key (extension "-trans") verbose force)
+ "Translate all *umls-files* to optimized import format."
+ (when verbose (format t "UMLS Import: Translating file ~A.~%" (fil (find-ufile "MRXW_NONENG.RRF"))))
+ (make-noneng-index-file extension :force force)
(dolist (f (remove "MRXW_NONENG.RRF" *umls-files* :test #'string= :key #'fil))
- (translate-umls-file f extension)))
+ (when verbose (format t "UMLS Import: Translating file ~A.~%" (fil f)))
+ (translate-umls-file f extension :force force)))
-(defun translate-umls-file (file extension)
+(defun translate-umls-file (file extension &key force)
"Translate a umls file into a format suitable for sql copy cmd"
- (translate-files file extension (list file)))
+ (translate-files file extension (list file) :force force))
-(defun make-noneng-index-file (extension)
+(defun make-noneng-index-file (extension &key force)
"Make non-english index file"
(translate-files (find-ufile "MRXW_NONENG.RRF")
- extension (noneng-lang-index-files)))
-
-(defun translate-files (out-ufile extension input-ufiles)
+ extension (noneng-lang-index-files) :force force))
+
+(defun verify-translation-file (output-path input-ufiles)
+ "Returns t if translation file exists and is correct size. Warns and deletes incomplete translation file."
+ (when (probe-file output-path)
+ (let ((translated-lines 0)
+ (input-lines 0)
+ (eof (cons nil nil)))
+ (catch 'done-counting
+ (with-open-file (ts output-path :direction :input
+ #+(and clisp unicode) :external-format
+ #+(and clisp unicode) charset:utf-8)
+ (do ()
+ ((eq (read-line ts nil eof) eof))
+ (incf translated-lines)))
+ (dolist (input-ufile input-ufiles)
+ (with-umls-ufile (line input-ufile)
+ (incf input-lines)
+ (when (> input-lines translated-lines)
+ (throw 'done-counting 'incomplete)))))
+ (cond
+ ((< input-lines translated-lines)
+ (format t "Translated file ~A incomplete, deleting...~%" output-path)
+ (delete-file output-path)
+ nil)
+ ((eql input-lines translated-lines)
+ (format t "Translated file ~A already exists: skipping...~%" output-path)
+ t)
+ ((eql input-lines 0)
+ (warn "The number of input lines is 0 for output file ~A." output-path)
+ nil)
+ ((> translated-lines input-lines)
+ (error "Shouldn't happen. Translated lines of ~A is ~D, greater than input lines ~D"
+ output-path translated-lines input-lines)
+ (delete-file output-path)
+ nil)))))
+
+(defun translate-files (out-ufile extension input-ufiles &key force)
"Translate a umls file into a format suitable for sql copy cmd"
(let ((output-path (ufile-pathname out-ufile extension)))
- (if (probe-file output-path)
- (format t "File ~A already exists: skipping~%" output-path)
- (with-open-file (ostream output-path :direction :output)
- (dolist (input-ufile input-ufiles)
- (with-umls-ufile (line input-ufile)
- (translate-line out-ufile line ostream)
- (princ #\newline ostream)))))))
+ (when (and (not force) (verify-translation-file output-path input-ufiles))
+ (return-from translate-files output-path))
+ (with-open-file (ostream output-path :direction :output
+ :if-exists :overwrite
+ :if-does-not-exist :create
+ #+(and clisp unicode) :external-format
+ #+(and clisp unicode) charset:utf-8)
+ (dolist (input-ufile input-ufiles)
+ (with-umls-ufile (line input-ufile)
+ (translate-line out-ufile line ostream)
+ (princ #\newline ostream))))))
(defun translate-line (file line strm)
"Translate a single line for sql output"
nil "COPY ~a FROM '~a' using delimiters '|' with null as ''"
(table file) (ufile-pathname file extension)))
-(defun mysql-copy-cmd (file extension &key local-file)
+(defun mysql-copy-cmd (file extension &key (local-file t))
"Return mysql copy statement for a file"
(format
nil
"LOAD DATA ~AINFILE \"~a\" INTO TABLE ~a FIELDS TERMINATED BY \"|\""
(if local-file "LOCAL " "")
- (ufile-pathname file extension) (table file)))
+ (namestring (ufile-pathname file extension)) (table file)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare (optimize (speed 3) (space 0)))
(ensure-ucols+ufiles)
(let ((max 0))
- (declare (fixnum max))
+ (declare (type (integer 0 1000000) max))
(dolist (ucol *umls-cols*)
- (when (> (cmax ucol) max)
+ (when (> (the (integer 0 1000000) (cmax ucol)) max)
(setq max (cmax ucol))))
max))
(let ((rowsizes '()))
(dolist (file *umls-files*)
(let ((row 0))
+ (declare (type (integer 0 1000000) row))
(dolist (ucol (ucols file))
- (incf row (1+ (cmax ucol))))
+ (let* ((col-max (cmax ucol))
+ (max-with-delim (1+ col-max)))
+ (declare (type (integer 0 1000000) col-max max-with-delim))
+ (incf row max-with-delim)))
(push row rowsizes)))
(car (sort rowsizes #'>))))
projects / umlisp.git / blobdiff