r11624: improve find-uconso-code

[umlisp.git] / create-sql.lisp

diff --git a/create-sql.lisp b/create-sql.lisp
index 02a8701d3ff5cd9a4941c9ae3035a75b97fc2bf2..04bc2b6bed3e0f676ddeb64761509a7360fd39e3 100644 (file)
--- a/create-sql.lisp
+++ b/create-sql.lisp
@@ -211,7 +211,8 @@
 (defun sql-create-indexes (conn &key (indexes +index-cols+) verbose)
   "SQL Databases: create all indexes"
   (dolist (idx indexes)
-    (when verbose (format t "UMLS Import: Creating index ~S.~%" idx))
+    (when verbose (format t "UMLS Import: Creating index for column ~A on table ~A.~%"
+                          (first idx) (second idx)))
     (ignore-errors (sql-execute (drop-index-cmd (car idx) (cadr idx)) conn))
     (sql-execute (create-index-cmd (car idx) (cadr idx) (caddr idx)) conn)))
 
@@ -243,16 +244,15 @@
     (sql-create-indexes conn :indexes +custom-index-cols+ :verbose verbose)
     (sql-create-special-tables conn)))
 
-(defun create-umls-db (&key (extension "-trans") (skip-translation nil) (verbose nil))
+(defun create-umls-db (&key (extension "-trans") (force-translation nil) (verbose nil))
   "SQL Databases: initializes entire database via SQL copy commands.
 This is much faster that using create-umls-db-insert."
   (when verbose (format t "UMLS Import: Starting.~%"))
   (ensure-ucols+ufiles)
   (when verbose (format t "UMLS Import: Preparsing files.~%"))
   (ensure-preparse)
-  (unless skip-translation
-    (when verbose (format t "UMLS Import: Converting text UMLS files to optimized format.~%"))
-    (translate-all-files extension))
+  (when verbose (format t "UMLS Import: Converting text UMLS files to optimized format.~%"))
+  (translate-all-files :extension extension :verbose verbose :force force-translation)
   (let ((copy-cmd
         (ecase (umls-sql-type)
           (:mysql #'mysql-copy-cmd)
@@ -262,7 +262,7 @@ This is much faster that using create-umls-db-insert."
       (sql-drop-tables conn)
       (sql-create-tables conn)
       (dolist (file *umls-files*)
-        (when verbose (format t "UMLS Import: Importing file ~A.~%" file))
+        (when verbose (format t "UMLS Import: Importing file ~A to SQL.~%" (fil file)))
        (sql-execute (funcall copy-cmd file extension) conn))
       (When verbose (format t "UMLS Import: Creating SQL indices.~%"))
       (sql-create-indexes conn :verbose verbose)
@@ -275,20 +275,22 @@ This is much faster that using create-umls-db-insert."
   (When verbose (format t "UMLS Import: Completed.~%"))
   t)
 
-(defun translate-all-files (&optional (extension "-trans"))
-  "Copy translated files and return postgresql copy commands to import"
-  (make-noneng-index-file extension)
+(defun translate-all-files (&key (extension "-trans") verbose force)
+  "Translate all *umls-files* to optimized import format."
+  (when verbose (format t "UMLS Import: Translating file ~A.~%" (fil (find-ufile "MRXW_NONENG.RRF"))))
+  (make-noneng-index-file extension :force force)
   (dolist (f (remove "MRXW_NONENG.RRF" *umls-files* :test #'string= :key #'fil))
-    (translate-umls-file f extension)))
+    (when verbose (format t "UMLS Import: Translating file ~A.~%" (fil f)))
+    (translate-umls-file f extension :force force)))
 
-(defun translate-umls-file (file extension)
+(defun translate-umls-file (file extension &key force)
   "Translate a umls file into a format suitable for sql copy cmd"
-  (translate-files file extension (list file)))
+  (translate-files file extension (list file) :force force))
 
-(defun make-noneng-index-file (extension)
+(defun make-noneng-index-file (extension &key force)
   "Make non-english index file"
   (translate-files (find-ufile "MRXW_NONENG.RRF")
-                  extension (noneng-lang-index-files)))
+                  extension (noneng-lang-index-files) :force force))
 
 (defun verify-translation-file (output-path input-ufiles)
   "Returns t if translation file exists and is correct size. Warns and deletes incomplete translation file."
@@ -325,12 +327,14 @@ This is much faster that using create-umls-db-insert."
           (delete-file output-path)
           nil)))))
 
-(defun translate-files (out-ufile extension input-ufiles)
+(defun translate-files (out-ufile extension input-ufiles &key force)
   "Translate a umls file into a format suitable for sql copy cmd"
   (let ((output-path (ufile-pathname out-ufile extension)))
-    (when (verify-translation-file output-path input-ufiles)
+    (when (and (not force) (verify-translation-file output-path input-ufiles))
       (return-from translate-files output-path))
     (with-open-file (ostream output-path :direction :output
+                             :if-exists :overwrite
+                             :if-does-not-exist :create
                              #+(and clisp unicode) :external-format
                              #+(and clisp unicode) charset:utf-8)
       (dolist (input-ufile input-ufiles)
@@ -419,9 +423,9 @@ This is much faster that using create-umls-db-insert."
   (declare (optimize (speed 3) (space 0)))
   (ensure-ucols+ufiles)
   (let ((max 0))
-    (declare (fixnum max))
+    (declare (type (integer 0 1000000) max))
     (dolist (ucol *umls-cols*)
-      (when (> (cmax ucol) max)
+      (when (> (the (integer 0 1000000) (cmax ucol)) max)
        (setq max (cmax ucol))))
     max))
 
@@ -432,7 +436,11 @@ This is much faster that using create-umls-db-insert."
   (let ((rowsizes '()))
     (dolist (file *umls-files*)
       (let ((row 0))
+        (declare (type (integer 0 1000000) row))
        (dolist (ucol (ucols file))
-         (incf row (1+ (cmax ucol))))
+          (let* ((col-max (cmax ucol))
+                 (max-with-delim (1+ col-max)))
+            (declare (type (integer 0 1000000) col-max max-with-delim))
+            (incf row max-with-delim)))
        (push row rowsizes)))
     (car (sort rowsizes #'>))))