1 ;;;; -*- Mode: LISP; Syntax: ANSI-Common-Lisp; Base: 10 -*-
2 ;;;; *************************************************************************
3 ;;;; FILE IDENTIFICATION
5 ;;;; Name: postgresql-socket.cl
6 ;;;; Purpose: Low-level PostgreSQL interface using sockets
7 ;;;; Programmers: Kevin M. Rosenberg based on
8 ;;;; Original code by Pierre R. Mai
10 ;;;; Date Started: Feb 2002
12 ;;;; $Id: postgresql-socket-api.cl,v 1.6 2002/03/26 14:12:12 kevin Exp $
14 ;;;; This file, part of CLSQL, is Copyright (c) 2002 by Kevin M. Rosenberg
15 ;;;; and Copyright (c) 1999-2001 by Pierre R. Mai
17 ;;;; CLSQL users are granted the rights to distribute and use this software
18 ;;;; as governed by the terms of the Lisp Lesser GNU Public License
19 ;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL.
20 ;;;; *************************************************************************
23 ;;;; Changes by Kevin Rosenberg
24 ;;;; - Added socket open functions for Allegro and Lispworks
25 ;;;; - Changed CMUCL FFI to UFFI
26 ;;;; - Added necessary (force-output) for socket streams on
27 ;;;; Allegro and Lispworks
28 ;;;; - Added initialization variable
29 ;;;; - Added field type processing
32 (declaim (optimize (debug 3) (speed 3) (safety 1) (compilation-speed 0)))
33 (in-package :postgresql-socket)
35 (uffi:def-enum pgsql-ftype
42 (defmethod database-type-library-loaded ((database-type
43 (eql :postgresql-socket)))
44 "T if foreign library was able to be loaded successfully. Always true for
51 (defmacro define-message-constants (description &rest clauses)
52 (assert (evenp (length clauses)))
53 (loop with seen-characters = nil
54 for (name char) on clauses by #'cddr
55 for char-code = (char-code char)
56 for doc-string = (format nil "~A (~:C): ~A" description char name)
57 if (member char seen-characters)
58 do (error "Duplicate message type ~@C for group ~A" char description)
61 `(defconstant ,name ,char-code ,doc-string)
63 and do (push char seen-characters)
65 (return `(progn ,@result-clauses))))
67 (eval-when (:compile-toplevel :load-toplevel :execute)
68 (define-message-constants "Backend Message Constants"
69 +ascii-row-message+ #\D
70 +authentication-message+ #\R
71 +backend-key-message+ #\K
72 +binary-row-message+ #\B
73 +completed-response-message+ #\C
74 +copy-in-response-message+ #\G
75 +copy-out-response-message+ #\H
76 +cursor-response-message+ #\P
77 +empty-query-response-message+ #\I
78 +error-response-message+ #\E
79 +function-response-message+ #\V
80 +notice-response-message+ #\N
81 +notification-response-message+ #\A
82 +ready-for-query-message+ #\Z
83 +row-description-message+ #\T))
85 (defgeneric send-socket-value (type socket value))
87 (defmethod send-socket-value ((type (eql 'int32)) socket (value integer))
88 (write-byte (ldb (byte 8 24) value) socket)
89 (write-byte (ldb (byte 8 16) value) socket)
90 (write-byte (ldb (byte 8 8) value) socket)
91 (write-byte (ldb (byte 8 0) value) socket))
93 (defmethod send-socket-value ((type (eql 'int16)) socket (value integer))
94 (write-byte (ldb (byte 8 8) value) socket)
95 (write-byte (ldb (byte 8 0) value) socket))
97 (defmethod send-socket-value ((type (eql 'int8)) socket (value integer))
98 (write-byte (ldb (byte 8 0) value) socket))
100 (defmethod send-socket-value ((type (eql 'string)) socket (value string))
101 (loop for char across value
102 for code = (char-code char)
103 do (write-byte code socket)
104 finally (write-byte 0 socket)))
106 (defmethod send-socket-value ((type (eql 'limstring)) socket (value string))
107 (loop for char across value
108 for code = (char-code char)
109 do (write-byte code socket)))
111 (defmethod send-socket-value ((type (eql 'byte)) socket (value integer))
112 (write-byte value socket))
114 (defmethod send-socket-value ((type (eql 'byte)) socket (value character))
115 (write-byte (char-code value) socket))
117 (defmethod send-socket-value ((type (eql 'byte)) socket value)
118 (write-sequence value socket))
120 (defgeneric read-socket-value (type socket))
122 (defmethod read-socket-value ((type (eql 'int32)) socket)
124 (setf (ldb (byte 8 24) result) (read-byte socket))
125 (setf (ldb (byte 8 16) result) (read-byte socket))
126 (setf (ldb (byte 8 8) result) (read-byte socket))
127 (setf (ldb (byte 8 0) result) (read-byte socket))
130 (defmethod read-socket-value ((type (eql 'int16)) socket)
132 (setf (ldb (byte 8 8) result) (read-byte socket))
133 (setf (ldb (byte 8 0) result) (read-byte socket))
136 (defmethod read-socket-value ((type (eql 'int8)) socket)
139 (defmethod read-socket-value ((type (eql 'string)) socket)
140 (with-output-to-string (out)
141 (loop for code = (read-byte socket)
143 do (write-char (code-char code) out))))
145 (defgeneric skip-socket-value (type socket))
147 (defmethod skip-socket-value ((type (eql 'int32)) socket)
148 (dotimes (i 4) (read-byte socket)))
150 (defmethod skip-socket-value ((type (eql 'int16)) socket)
151 (dotimes (i 2) (read-byte socket)))
153 (defmethod skip-socket-value ((type (eql 'int8)) socket)
156 (defmethod skip-socket-value ((type (eql 'string)) socket)
157 (loop until (zerop (read-byte socket))))
159 (defmacro define-message-sender (name (&rest args) &rest clauses)
160 (loop with socket-var = (gensym)
161 for (type value) in clauses
163 `(send-socket-value ',type ,socket-var ,value)
167 `(defun ,name (,socket-var ,@args)
170 (defun pad-limstring (string limit)
171 (let ((result (make-string limit :initial-element #\NULL)))
172 (loop for char across string
173 for index from 0 below limit
174 do (setf (char result index) char))
177 (define-message-sender send-startup-message
178 (database user &optional (command-line "") (backend-tty ""))
180 (int32 #x00020000) ; Version 2.0
181 (limstring (pad-limstring database 64))
182 (limstring (pad-limstring user 32))
183 (limstring (pad-limstring command-line 64))
184 (limstring (pad-limstring "" 64)) ; Unused
185 (limstring (pad-limstring backend-tty 64)))
187 (define-message-sender send-terminate-message ()
190 (define-message-sender send-unencrypted-password-message (password)
191 (int32 (+ 5 (length password)))
194 (define-message-sender send-query-message (query)
198 (define-message-sender send-encrypted-password-message (crypted-password)
199 (int32 (+ 5 (length crypted-password)))
200 (string crypted-password))
202 (define-message-sender send-cancel-request (pid key)
204 (int32 80877102) ; Magic
209 (defun read-socket-sequence (string stream)
210 "KMR -- Added to support reading from binary stream into a string"
211 (declare (optimize (speed 3) (safety 0))
213 (dotimes (i (length string))
215 (setf (char string i) (code-char (read-byte stream))))
219 ;;; Support for encrypted password transmission
221 (defconstant +crypt-library+ "/usr/lib/libcrypt.so"
222 "Name of the shared library to load in order to access the crypt
223 function named by `*crypt-function-name*'.")
225 (defvar *crypt-library-loaded* nil)
227 (defun crypt-password (password salt)
228 "Encrypt a password for transmission to a PostgreSQL server."
229 (unless *crypt-library-loaded*
230 (uffi:load-foreign-library +crypt-library+ :supporting-libaries '("c"))
231 (eval (uffi:def-function "crypt"
234 :returning :cstring))
235 (setq *crypt-library-loaded* t))
236 (uffi:with-cstring (password-cstring password)
237 (uffi:with-cstring (salt-cstring salt)
238 (uffi:convert-from-cstring (crypt password-cstring salt-cstring)))))
239 ;;; Condition hierarchy
241 (define-condition postgresql-condition (condition)
242 ((connection :initarg :connection :reader postgresql-condition-connection)
243 (message :initarg :message :reader postgresql-condition-message))
246 (format stream "~@<~A occurred on connection ~A. ~:@_Reason: ~A~:@>"
248 (postgresql-condition-connection c)
249 (postgresql-condition-message c)))))
251 (define-condition postgresql-error (error postgresql-condition)
254 (define-condition postgresql-fatal-error (postgresql-error)
257 (define-condition postgresql-login-error (postgresql-fatal-error)
260 (define-condition postgresql-warning (warning postgresql-condition)
263 (define-condition postgresql-notification (postgresql-condition)
267 (format stream "~@<Asynchronous notification on connection ~A: ~:@_~A~:@>"
268 (postgresql-condition-connection c)
269 (postgresql-condition-message c)))))
273 (defstruct postgresql-connection
285 (defstruct postgresql-cursor
292 (defconstant +postgresql-server-default-port+ 5432
293 "Default port of PostgreSQL server.")
295 (defvar *postgresql-server-socket-timeout* 60
296 "Timeout in seconds for reads from the PostgreSQL server.")
300 (defun open-postgresql-socket (host port)
303 ;; Directory to unix-domain socket
304 (ext:connect-to-unix-socket
306 (make-pathname :name ".s.PGSQL" :type (princ-to-string port)
309 (ext:connect-to-inet-socket host port))))
312 (defun open-postgresql-socket-stream (host port)
313 (system:make-fd-stream
314 (open-postgresql-socket host port)
315 :input t :output t :element-type '(unsigned-byte 8)
317 :timeout *postgresql-server-socket-timeout*))
320 (defun open-postgresql-socket-stream (host port)
323 (let ((path (namestring
324 (make-pathname :name ".s.PGSQL" :type (princ-to-string port)
326 (socket:make-socket :type :stream :address-family :file
328 :remote-filename path :local-filename path)))
330 (socket:with-pending-connect
331 (mp:with-timeout (*postgresql-server-socket-timeout* (error "connect failed"))
332 (socket:make-socket :type :stream :address-family :internet
333 :remote-port port :remote-host host
334 :connect :active :nodelay t))))
338 (defun open-postgresql-socket-stream (host port)
341 (error "File sockets not supported on Lispworks."))
343 (comm:open-tcp-stream host port :direction :io :element-type '(unsigned-byte 8)
344 :read-timeout *postgresql-server-socket-timeout*))
347 ;;; Interface Functions
349 (defun open-postgresql-connection (&key (host (cmucl-compat:required-argument))
350 (port +postgresql-server-default-port+)
351 (database (cmucl-compat:required-argument))
352 (user (cmucl-compat:required-argument))
353 options tty password)
354 "Open a connection to a PostgreSQL server with the given parameters.
355 Note that host, database and user arguments must be supplied.
357 If host is a pathname, it is assumed to name a directory containing
358 the local unix-domain sockets of the server, with port selecting which
359 of those sockets to open. If host is a string, it is assumed to be
360 the name of the host running the PostgreSQL server. In that case a
361 TCP connection to the given port on that host is opened in order to
362 communicate with the server. In either case the port argument
363 defaults to `+postgresql-server-default-port+'.
365 Password is the clear-text password to be passed in the authentication
366 phase to the server. Depending on the server set-up, it is either
367 passed in the clear, or encrypted via crypt and a server-supplied
368 salt. In that case the alien function specified by `*crypt-library*'
369 and `*crypt-function-name*' is used for encryption.
371 Note that all the arguments (including the clear-text password
372 argument) are stored in the `postgresql-connection' structure, in
373 order to facilitate automatic reconnection in case of communication
375 (reopen-postgresql-connection
376 (make-postgresql-connection :host host :port port
377 :options (or options "") :tty (or tty "")
378 :database database :user user
379 :password (or password ""))))
381 (defun reopen-postgresql-connection (connection)
382 "Reopen the given PostgreSQL connection. Closes any existing
383 connection, if it is still open."
384 (when (postgresql-connection-open-p connection)
385 (close-postgresql-connection connection))
386 (let ((socket (open-postgresql-socket-stream
387 (postgresql-connection-host connection)
388 (postgresql-connection-port connection))))
391 (setf (postgresql-connection-socket connection) socket)
392 (send-startup-message socket
393 (postgresql-connection-database connection)
394 (postgresql-connection-user connection)
395 (postgresql-connection-options connection)
396 (postgresql-connection-tty connection))
397 (force-output socket)
399 (case (read-socket-value 'int8 socket)
400 (#.+authentication-message+
401 (case (read-socket-value 'int32 socket)
404 (error 'postgresql-login-error
405 :connection connection
407 "Postmaster expects unsupported Kerberos authentication."))
409 (send-unencrypted-password-message
411 (postgresql-connection-password connection)))
413 (let ((salt (make-string 2)))
414 (read-socket-sequence salt socket)
415 (send-encrypted-password-message
418 (postgresql-connection-password connection) salt))))
420 (error 'postgresql-login-error
421 :connection connection
423 "Postmaster expects unknown authentication method."))))
424 (#.+error-response-message+
425 (let ((message (read-socket-value 'string socket)))
426 (error 'postgresql-login-error
427 :connection connection :message message)))
429 (error 'postgresql-login-error
430 :connection connection
432 "Received garbled message from Postmaster"))))
433 ;; Start backend communication
434 (force-output socket)
436 (case (read-socket-value 'int8 socket)
437 (#.+backend-key-message+
438 (setf (postgresql-connection-pid connection)
439 (read-socket-value 'int32 socket)
440 (postgresql-connection-key connection)
441 (read-socket-value 'int32 socket)))
442 (#.+ready-for-query-message+
445 (#.+error-response-message+
446 (let ((message (read-socket-value 'string socket)))
447 (error 'postgresql-login-error
448 :connection connection
450 (#.+notice-response-message+
451 (let ((message (read-socket-value 'string socket)))
452 (warn 'postgresql-warning :connection connection
455 (error 'postgresql-login-error
456 :connection connection
458 "Received garbled message from Postmaster")))))
462 (defun close-postgresql-connection (connection &optional abort)
465 (send-terminate-message (postgresql-connection-socket connection))))
466 (close (postgresql-connection-socket connection)))
468 (defun postgresql-connection-open-p (connection)
469 (let ((socket (postgresql-connection-socket connection)))
470 (and socket (streamp socket) (open-stream-p socket))))
472 (defun ensure-open-postgresql-connection (connection)
473 (unless (postgresql-connection-open-p connection)
474 (reopen-postgresql-connection connection)))
476 (defun process-async-messages (connection)
477 (assert (postgresql-connection-open-p connection))
478 ;; Process any asnychronous messages
479 (loop with socket = (postgresql-connection-socket connection)
480 while (listen socket)
482 (case (read-socket-value 'int8 socket)
483 (#.+notice-response-message+
484 (let ((message (read-socket-value 'string socket)))
485 (warn 'postgresql-warning :connection connection
487 (#.+notification-response-message+
488 (let ((pid (read-socket-value 'int32 socket))
489 (message (read-socket-value 'string socket)))
490 (when (= pid (postgresql-connection-pid connection))
491 (signal 'postgresql-notification :connection connection
494 (close-postgresql-connection connection)
495 (error 'postgresql-fatal-error :connection connection
496 :message "Received garbled message from backend")))))
498 (defun start-query-execution (connection query)
499 (ensure-open-postgresql-connection connection)
500 (process-async-messages connection)
501 (send-query-message (postgresql-connection-socket connection) query)
502 (force-output (postgresql-connection-socket connection)))
504 (defun wait-for-query-results (connection)
505 (assert (postgresql-connection-open-p connection))
506 (let ((socket (postgresql-connection-socket connection))
510 (case (read-socket-value 'int8 socket)
511 (#.+completed-response-message+
512 (return (values :completed (read-socket-value 'string socket))))
513 (#.+cursor-response-message+
514 (setq cursor-name (read-socket-value 'string socket)))
515 (#.+row-description-message+
516 (let* ((count (read-socket-value 'int16 socket))
521 (read-socket-value 'string socket)
522 (read-socket-value 'int32 socket)
523 (read-socket-value 'int16 socket)
524 (read-socket-value 'int32 socket)))))
527 (make-postgresql-cursor :connection connection
530 (#.+copy-in-response-message+
532 (#.+copy-out-response-message+
534 (#.+ready-for-query-message+
538 (#.+error-response-message+
539 (let ((message (read-socket-value 'string socket)))
541 (make-condition 'postgresql-error
542 :connection connection :message message))))
543 (#.+notice-response-message+
544 (let ((message (read-socket-value 'string socket)))
545 (warn 'postgresql-warning
546 :connection connection :message message)))
547 (#.+notification-response-message+
548 (let ((pid (read-socket-value 'int32 socket))
549 (message (read-socket-value 'string socket)))
550 (when (= pid (postgresql-connection-pid connection))
551 (signal 'postgresql-notification :connection connection
554 (close-postgresql-connection connection)
555 (error 'postgresql-fatal-error :connection connection
556 :message "Received garbled message from backend"))))))
558 (defun read-null-bit-vector (socket count)
559 (let ((result (make-array count :element-type 'bit)))
560 (dotimes (offset (ceiling count 8))
561 (loop with byte = (read-byte socket)
562 for index from (* offset 8) below (min count (* (1+ offset) 8))
563 for weight downfrom 7
564 do (setf (aref result index) (ldb (byte 1 weight) byte))))
568 (defun read-field (socket type)
569 (let* ((length (- (read-socket-value 'int32 socket) 4)))
572 (read-integer-from-socket socket length))
574 (read-double-from-socket socket length))
576 (let ((result (make-string length)))
577 (read-socket-sequence result socket)
580 (uffi:def-constant +char-code-zero+ (char-code #\0))
581 (uffi:def-constant +char-code-minus+ (char-code #\-))
582 (uffi:def-constant +char-code-plus+ (char-code #\+))
583 (uffi:def-constant +char-code-period+ (char-code #\.))
585 (defun read-integer-from-socket (socket length)
587 (first-char (read-byte socket))
589 (declare (fixnum first-char))
590 (if (eql first-char +char-code-minus+)
592 (setq val (- first-char +char-code-zero+)))
593 (dotimes (i (1- length))
597 (- (read-byte socket) +char-code-zero+))))
602 (defmacro ascii-digit (int)
603 (let ((offset (gensym)))
604 `(let ((,offset (- ,int +char-code-zero+)))
605 (declare (fixnum ,int ,offset))
606 (if (and (plusp ,offset)
612 (defun read-double-from-socket (socket length)
613 (let ((before-decimal 0)
617 (char (read-byte socket))
620 (declare (fixnum first-char))
622 ((eql char +char-code-minus+)
624 (setq char (read-byte socket))
626 ((eql char +char-code-plus+)
627 (setq char (read-byte socket))
630 (dotimes (i (1- length))
631 (let ((weight (ascii-digit char)))
633 ((and weight (not decimalp)) ;; before decimal point
634 (setq before-decimal (+ weight (* 10 before-decimal))))
635 ((and weight decimalp) ;; after decimal point
636 (setq after-decimal (+ weight (* 10 after-decimal)))
637 (incf decimal-count))
638 ((and (eql char +char-code-period+) decimalp)
640 ((or (eql char +char-code-e+) ;; E is for exponent
641 (eql char +char-code-upper-e+))
642 (multiple-value-bind (num idx)
643 (parse-integer string :start (1+ index) :end end
644 :radix radix :junk-allowed junk-allowed)
645 (setq exponent (or num 0)
647 (when (= index end) (return nil))))
649 (setq char (read-byte socket))))
657 (defun read-double-from-socket (socket length)
658 (let ((result (make-string length)))
659 (read-socket-sequence result socket)
660 (let ((*read-default-float-format* 'double-float))
661 (read-from-string result))))
663 (defun read-cursor-row (cursor types)
664 (let* ((connection (postgresql-cursor-connection cursor))
665 (socket (postgresql-connection-socket connection))
666 (fields (postgresql-cursor-fields cursor)))
667 (assert (postgresql-connection-open-p connection))
669 (let ((code (read-socket-value 'int8 socket)))
671 (#.+ascii-row-message+
673 (loop with count = (length fields)
674 with null-vector = (read-null-bit-vector socket count)
676 for null-bit across null-vector
678 for null-p = (zerop null-bit)
683 (read-field socket (nth i types)))))
684 (#.+binary-row-message+
686 (#.+completed-response-message+
687 (return (values nil (read-socket-value 'string socket))))
688 (#.+error-response-message+
689 (let ((message (read-socket-value 'string socket)))
690 (error 'postgresql-error
691 :connection connection :message message)))
692 (#.+notice-response-message+
693 (let ((message (read-socket-value 'string socket)))
694 (warn 'postgresql-warning
695 :connection connection :message message)))
696 (#.+notification-response-message+
697 (let ((pid (read-socket-value 'int32 socket))
698 (message (read-socket-value 'string socket)))
699 (when (= pid (postgresql-connection-pid connection))
700 (signal 'postgresql-notification :connection connection
703 (close-postgresql-connection connection)
704 (error 'postgresql-fatal-error :connection connection
705 :message "Received garbled message from backend")))))))
707 (defun map-into-indexed (result-seq func seq)
708 (dotimes (i (length seq))
710 (setf (elt result-seq i)
711 (funcall func (elt seq i) i)))
714 (defun copy-cursor-row (cursor sequence types)
715 (let* ((connection (postgresql-cursor-connection cursor))
716 (socket (postgresql-connection-socket connection))
717 (fields (postgresql-cursor-fields cursor)))
718 (assert (= (length fields) (length sequence)))
720 (let ((code (read-socket-value 'int8 socket)))
722 (#.+ascii-row-message+
725 (let* ((count (length sequence))
726 (null-vector (read-null-bit-vector socket count)))
729 (if (zerop (elt null-vector i))
730 (setf (elt sequence i) nil)
731 (let ((value (read-field socket (nth i types))))
732 (setf (elt sequence i) value)))))
735 #'(lambda (null-bit i)
738 (read-field socket (nth i types))))
739 (read-null-bit-vector socket (length sequence)))))
740 (#.+binary-row-message+
742 (#.+completed-response-message+
743 (return (values nil (read-socket-value 'string socket))))
744 (#.+error-response-message+
745 (let ((message (read-socket-value 'string socket)))
746 (error 'postgresql-error
747 :connection connection :message message)))
748 (#.+notice-response-message+
749 (let ((message (read-socket-value 'string socket)))
750 (warn 'postgresql-warning
751 :connection connection :message message)))
752 (#.+notification-response-message+
753 (let ((pid (read-socket-value 'int32 socket))
754 (message (read-socket-value 'string socket)))
755 (when (= pid (postgresql-connection-pid connection))
756 (signal 'postgresql-notification :connection connection
759 (close-postgresql-connection connection)
760 (error 'postgresql-fatal-error :connection connection
761 :message "Received garbled message from backend")))))))
763 (defun skip-cursor-row (cursor)
764 (let* ((connection (postgresql-cursor-connection cursor))
765 (socket (postgresql-connection-socket connection))
766 (fields (postgresql-cursor-fields cursor)))
768 (let ((code (read-socket-value 'int8 socket)))
770 (#.+ascii-row-message+
771 (loop for null-bit across
772 (read-null-bit-vector socket (length fields))
774 (unless (zerop null-bit)
775 (let* ((length (read-socket-value 'int32 socket)))
776 (loop repeat (- length 4) do (read-byte socket)))))
778 (#.+binary-row-message+
780 (#.+completed-response-message+
781 (return (values nil (read-socket-value 'string socket))))
782 (#.+error-response-message+
783 (let ((message (read-socket-value 'string socket)))
784 (error 'postgresql-error
785 :connection connection :message message)))
786 (#.+notice-response-message+
787 (let ((message (read-socket-value 'string socket)))
788 (warn 'postgresql-warning
789 :connection connection :message message)))
790 (#.+notification-response-message+
791 (let ((pid (read-socket-value 'int32 socket))
792 (message (read-socket-value 'string socket)))
793 (when (= pid (postgresql-connection-pid connection))
794 (signal 'postgresql-notification :connection connection
797 (close-postgresql-connection connection)
798 (error 'postgresql-fatal-error :connection connection
799 :message "Received garbled message from backend")))))))
801 (defun run-query (connection query &optional (types nil))
802 (start-query-execution connection query)
803 (multiple-value-bind (status cursor)
804 (wait-for-query-results connection)
805 (assert (eq status :cursor))
806 (loop for row = (read-cursor-row cursor types)
810 (wait-for-query-results connection))))