1 ;;;; -*- Mode: LISP; Syntax: ANSI-Common-Lisp; Base: 10 -*-
2 ;;;; *************************************************************************
3 ;;;; FILE IDENTIFICATION
5 ;;;; Name: postgresql-socket-api.lisp
6 ;;;; Purpose: Low-level PostgreSQL interface using sockets
7 ;;;; Programmers: Kevin M. Rosenberg based on
8 ;;;; Original code by Pierre R. Mai
10 ;;;; Date Started: Feb 2002
12 ;;;; $Id: postgresql-socket-api.lisp,v 1.2 2002/10/21 07:45:50 kevin Exp $
14 ;;;; This file, part of CLSQL, is Copyright (c) 2002 by Kevin M. Rosenberg
15 ;;;; and Copyright (c) 1999-2001 by Pierre R. Mai
17 ;;;; CLSQL users are granted the rights to distribute and use this software
18 ;;;; as governed by the terms of the Lisp Lesser GNU Public License
19 ;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL.
20 ;;;; *************************************************************************
23 ;;;; Changes by Kevin Rosenberg
24 ;;;; - Added socket open functions for Allegro and Lispworks
25 ;;;; - Changed CMUCL FFI to UFFI
26 ;;;; - Added necessary (force-output) for socket streams on
27 ;;;; Allegro and Lispworks
28 ;;;; - Added initialization variable
29 ;;;; - Added field type processing
32 (declaim (optimize (debug 3) (speed 3) (safety 1) (compilation-speed 0)))
33 (in-package :postgresql-socket)
35 (uffi:def-enum pgsql-ftype
43 (defmethod database-type-library-loaded ((database-type
44 (eql :postgresql-socket)))
45 "T if foreign library was able to be loaded successfully. Always true for
52 (defmacro define-message-constants (description &rest clauses)
53 (assert (evenp (length clauses)))
54 (loop with seen-characters = nil
55 for (name char) on clauses by #'cddr
56 for char-code = (char-code char)
57 for doc-string = (format nil "~A (~:C): ~A" description char name)
58 if (member char seen-characters)
59 do (error "Duplicate message type ~@C for group ~A" char description)
62 `(defconstant ,name ,char-code ,doc-string)
64 and do (push char seen-characters)
66 (return `(progn ,@result-clauses))))
68 (eval-when (:compile-toplevel :load-toplevel :execute)
69 (define-message-constants "Backend Message Constants"
70 +ascii-row-message+ #\D
71 +authentication-message+ #\R
72 +backend-key-message+ #\K
73 +binary-row-message+ #\B
74 +completed-response-message+ #\C
75 +copy-in-response-message+ #\G
76 +copy-out-response-message+ #\H
77 +cursor-response-message+ #\P
78 +empty-query-response-message+ #\I
79 +error-response-message+ #\E
80 +function-response-message+ #\V
81 +notice-response-message+ #\N
82 +notification-response-message+ #\A
83 +ready-for-query-message+ #\Z
84 +row-description-message+ #\T))
87 (declaim (inline read-byte write-byte))
89 (defun send-socket-value-int32 (socket value)
90 (declare (type stream socket)
91 (type (unsigned-byte 32) value))
92 (write-byte (ldb (byte 8 24) value) socket)
93 (write-byte (ldb (byte 8 16) value) socket)
94 (write-byte (ldb (byte 8 8) value) socket)
95 (write-byte (ldb (byte 8 0) value) socket)
98 (defun send-socket-value-int16 (socket value)
99 (declare (type stream socket)
100 (type (unsigned-byte 16) value))
101 (write-byte (ldb (byte 8 8) value) socket)
102 (write-byte (ldb (byte 8 0) value) socket)
105 (defun send-socket-value-int8 (socket value)
106 (declare (type stream socket)
107 (type (unsigned-byte 8) value))
108 (write-byte (ldb (byte 8 0) value) socket)
111 (defun send-socket-value-char-code (socket value)
112 (declare (type stream socket)
113 (type character value))
114 (write-byte (ldb (byte 8 0) (char-code value)) socket)
117 (defun send-socket-value-string (socket value)
118 (declare (type stream socket)
120 (loop for char across value
121 for code = (char-code char)
122 do (write-byte code socket)
123 finally (write-byte 0 socket))
126 (defun send-socket-value-limstring (socket value limit)
127 (declare (type stream socket)
130 (let ((length (length value)))
131 (dotimes (i (min length limit))
132 (let ((code (char-code (char value i))))
133 (write-byte code socket)))
134 (dotimes (i (- limit length))
135 (write-byte 0 socket)))
139 (defun read-socket-value-int32 (socket)
140 (declare (type stream socket))
141 (declare (optimize (speed 3)))
143 (declare (type (unsigned-byte 32) result))
144 (setf (ldb (byte 8 24) result) (read-byte socket))
145 (setf (ldb (byte 8 16) result) (read-byte socket))
146 (setf (ldb (byte 8 8) result) (read-byte socket))
147 (setf (ldb (byte 8 0) result) (read-byte socket))
150 (defun read-socket-value-int16 (socket)
151 (declare (type stream socket))
153 (declare (type (unsigned-byte 16) result))
154 (setf (ldb (byte 8 8) result) (read-byte socket))
155 (setf (ldb (byte 8 0) result) (read-byte socket))
158 (defun read-socket-value-int8 (socket)
159 (declare (type stream socket))
162 (defun read-socket-value-string (socket)
163 (declare (type stream socket))
164 (with-output-to-string (out)
165 (loop for code = (read-byte socket)
167 do (write-char (code-char code) out))))
170 (defmacro define-message-sender (name (&rest args) &rest clauses)
171 (let ((socket-var (gensym))
173 (dolist (clause clauses)
174 (let* ((type (first clause))
175 (fn (intern (concatenate 'string (symbol-name '#:send-socket-value-)
176 (symbol-name type)))))
177 (push `(,fn ,socket-var ,@(rest clause)) body)))
178 `(defun ,name (,socket-var ,@args)
181 (define-message-sender send-startup-message
182 (database user &optional (command-line "") (backend-tty ""))
184 (int32 #x00020000) ; Version 2.0
185 (limstring database 64)
187 (limstring command-line 64)
188 (limstring "" 64) ; Unused
189 (limstring backend-tty 64))
191 (define-message-sender send-terminate-message ()
194 (define-message-sender send-unencrypted-password-message (password)
195 (int32 (+ 5 (length password)))
198 (define-message-sender send-query-message (query)
202 (define-message-sender send-encrypted-password-message (crypted-password)
203 (int32 (+ 5 (length crypted-password)))
204 (string crypted-password))
206 (define-message-sender send-cancel-request (pid key)
208 (int32 80877102) ; Magic
213 (defun read-socket-sequence (string stream)
214 "KMR -- Added to support reading from binary stream into a string"
215 (declare (string string)
217 (optimize (speed 3) (safety 0)))
218 (dotimes (i (length string))
220 (setf (char string i) (code-char (read-byte stream))))
224 ;;; Support for encrypted password transmission
227 (eval-when (compile eval load)
228 (defvar *crypt-library-loaded* nil)
230 (unless *crypt-library-loaded*
231 (uffi:load-foreign-library
232 (uffi:find-foreign-library "libcrypt"
233 '("/usr/lib/" "/usr/local/lib/" "/lib/"))
234 :supporting-libraries '("c"))
235 (setq *crypt-library-loaded* t)))
237 (in-package :postgresql-socket)
239 (uffi:def-function "crypt"
244 (defun crypt-password (password salt)
245 "Encrypt a password for transmission to a PostgreSQL server."
246 (uffi:with-cstring (password-cstring password)
247 (uffi:with-cstring (salt-cstring salt)
248 (uffi:convert-from-cstring
249 (crypt password-cstring salt-cstring)))))
252 ;;;; Condition hierarchy
254 (define-condition postgresql-condition (condition)
255 ((connection :initarg :connection :reader postgresql-condition-connection)
256 (message :initarg :message :reader postgresql-condition-message))
259 (format stream "~@<~A occurred on connection ~A. ~:@_Reason: ~A~:@>"
261 (postgresql-condition-connection c)
262 (postgresql-condition-message c)))))
264 (define-condition postgresql-error (error postgresql-condition)
267 (define-condition postgresql-fatal-error (postgresql-error)
270 (define-condition postgresql-login-error (postgresql-fatal-error)
273 (define-condition postgresql-warning (warning postgresql-condition)
276 (define-condition postgresql-notification (postgresql-condition)
280 (format stream "~@<Asynchronous notification on connection ~A: ~:@_~A~:@>"
281 (postgresql-condition-connection c)
282 (postgresql-condition-message c)))))
286 (defstruct postgresql-connection
298 (defstruct postgresql-cursor
305 (defconstant +postgresql-server-default-port+ 5432
306 "Default port of PostgreSQL server.")
308 (defvar *postgresql-server-socket-timeout* 60
309 "Timeout in seconds for reads from the PostgreSQL server.")
313 (defun open-postgresql-socket (host port)
316 ;; Directory to unix-domain socket
317 (ext:connect-to-unix-socket
319 (make-pathname :name ".s.PGSQL" :type (princ-to-string port)
322 (ext:connect-to-inet-socket host port))))
325 (defun open-postgresql-socket-stream (host port)
326 (system:make-fd-stream
327 (open-postgresql-socket host port)
328 :input t :output t :element-type '(unsigned-byte 8)
330 :timeout *postgresql-server-socket-timeout*))
333 (defun open-postgresql-socket-stream (host port)
336 (let ((path (namestring
337 (make-pathname :name ".s.PGSQL" :type (princ-to-string port)
339 (socket:make-socket :type :stream :address-family :file
341 :remote-filename path :local-filename path)))
343 (socket:with-pending-connect
344 (mp:with-timeout (*postgresql-server-socket-timeout* (error "connect failed"))
345 (socket:make-socket :type :stream :address-family :internet
346 :remote-port port :remote-host host
347 :connect :active :nodelay t))))
351 (defun open-postgresql-socket-stream (host port)
354 (error "File sockets not supported on Lispworks."))
356 (comm:open-tcp-stream host port :direction :io :element-type '(unsigned-byte 8)
357 :read-timeout *postgresql-server-socket-timeout*))
360 ;;; Interface Functions
362 (defun open-postgresql-connection (&key (host (cmucl-compat:required-argument))
363 (port +postgresql-server-default-port+)
364 (database (cmucl-compat:required-argument))
365 (user (cmucl-compat:required-argument))
366 options tty password)
367 "Open a connection to a PostgreSQL server with the given parameters.
368 Note that host, database and user arguments must be supplied.
370 If host is a pathname, it is assumed to name a directory containing
371 the local unix-domain sockets of the server, with port selecting which
372 of those sockets to open. If host is a string, it is assumed to be
373 the name of the host running the PostgreSQL server. In that case a
374 TCP connection to the given port on that host is opened in order to
375 communicate with the server. In either case the port argument
376 defaults to `+postgresql-server-default-port+'.
378 Password is the clear-text password to be passed in the authentication
379 phase to the server. Depending on the server set-up, it is either
380 passed in the clear, or encrypted via crypt and a server-supplied
381 salt. In that case the alien function specified by `*crypt-library*'
382 and `*crypt-function-name*' is used for encryption.
384 Note that all the arguments (including the clear-text password
385 argument) are stored in the `postgresql-connection' structure, in
386 order to facilitate automatic reconnection in case of communication
388 (reopen-postgresql-connection
389 (make-postgresql-connection :host host :port port
390 :options (or options "") :tty (or tty "")
391 :database database :user user
392 :password (or password ""))))
394 (defun reopen-postgresql-connection (connection)
395 "Reopen the given PostgreSQL connection. Closes any existing
396 connection, if it is still open."
397 (when (postgresql-connection-open-p connection)
398 (close-postgresql-connection connection))
399 (let ((socket (open-postgresql-socket-stream
400 (postgresql-connection-host connection)
401 (postgresql-connection-port connection))))
404 (setf (postgresql-connection-socket connection) socket)
405 (send-startup-message socket
406 (postgresql-connection-database connection)
407 (postgresql-connection-user connection)
408 (postgresql-connection-options connection)
409 (postgresql-connection-tty connection))
410 (force-output socket)
412 (case (read-socket-value-int8 socket)
413 (#.+authentication-message+
414 (case (read-socket-value-int32 socket)
417 (error 'postgresql-login-error
418 :connection connection
420 "Postmaster expects unsupported Kerberos authentication."))
422 (send-unencrypted-password-message
424 (postgresql-connection-password connection)))
426 (let ((salt (make-string 2)))
427 (read-socket-sequence salt socket)
428 (send-encrypted-password-message
431 (postgresql-connection-password connection) salt))))
433 (error 'postgresql-login-error
434 :connection connection
436 "Postmaster expects unknown authentication method."))))
437 (#.+error-response-message+
438 (let ((message (read-socket-value-string socket)))
439 (error 'postgresql-login-error
440 :connection connection :message message)))
442 (error 'postgresql-login-error
443 :connection connection
445 "Received garbled message from Postmaster"))))
446 ;; Start backend communication
447 (force-output socket)
449 (case (read-socket-value-int8 socket)
450 (#.+backend-key-message+
451 (setf (postgresql-connection-pid connection)
452 (read-socket-value-int32 socket)
453 (postgresql-connection-key connection)
454 (read-socket-value-int32 socket)))
455 (#.+ready-for-query-message+
458 (#.+error-response-message+
459 (let ((message (read-socket-value-string socket)))
460 (error 'postgresql-login-error
461 :connection connection
463 (#.+notice-response-message+
464 (let ((message (read-socket-value-string socket)))
465 (warn 'postgresql-warning :connection connection
468 (error 'postgresql-login-error
469 :connection connection
471 "Received garbled message from Postmaster")))))
475 (defun close-postgresql-connection (connection &optional abort)
478 (send-terminate-message (postgresql-connection-socket connection))))
479 (close (postgresql-connection-socket connection)))
481 (defun postgresql-connection-open-p (connection)
482 (let ((socket (postgresql-connection-socket connection)))
483 (and socket (streamp socket) (open-stream-p socket))))
485 (defun ensure-open-postgresql-connection (connection)
486 (unless (postgresql-connection-open-p connection)
487 (reopen-postgresql-connection connection)))
489 (defun process-async-messages (connection)
490 (assert (postgresql-connection-open-p connection))
491 ;; Process any asnychronous messages
492 (loop with socket = (postgresql-connection-socket connection)
493 while (listen socket)
495 (case (read-socket-value-int8 socket)
496 (#.+notice-response-message+
497 (let ((message (read-socket-value-string socket)))
498 (warn 'postgresql-warning :connection connection
500 (#.+notification-response-message+
501 (let ((pid (read-socket-value-int32 socket))
502 (message (read-socket-value-string socket)))
503 (when (= pid (postgresql-connection-pid connection))
504 (signal 'postgresql-notification :connection connection
507 (close-postgresql-connection connection)
508 (error 'postgresql-fatal-error :connection connection
509 :message "Received garbled message from backend")))))
511 (defun start-query-execution (connection query)
512 (ensure-open-postgresql-connection connection)
513 (process-async-messages connection)
514 (send-query-message (postgresql-connection-socket connection) query)
515 (force-output (postgresql-connection-socket connection)))
517 (defun wait-for-query-results (connection)
518 (assert (postgresql-connection-open-p connection))
519 (let ((socket (postgresql-connection-socket connection))
523 (case (read-socket-value-int8 socket)
524 (#.+completed-response-message+
525 (return (values :completed (read-socket-value-string socket))))
526 (#.+cursor-response-message+
527 (setq cursor-name (read-socket-value-string socket)))
528 (#.+row-description-message+
529 (let* ((count (read-socket-value-int16 socket))
534 (read-socket-value-string socket)
535 (read-socket-value-int32 socket)
536 (read-socket-value-int16 socket)
537 (read-socket-value-int32 socket)))))
540 (make-postgresql-cursor :connection connection
543 (#.+copy-in-response-message+
545 (#.+copy-out-response-message+
547 (#.+ready-for-query-message+
551 (#.+error-response-message+
552 (let ((message (read-socket-value-string socket)))
554 (make-condition 'postgresql-error
555 :connection connection :message message))))
556 (#.+notice-response-message+
557 (let ((message (read-socket-value-string socket)))
558 (warn 'postgresql-warning
559 :connection connection :message message)))
560 (#.+notification-response-message+
561 (let ((pid (read-socket-value-int32 socket))
562 (message (read-socket-value-string socket)))
563 (when (= pid (postgresql-connection-pid connection))
564 (signal 'postgresql-notification :connection connection
567 (close-postgresql-connection connection)
568 (error 'postgresql-fatal-error :connection connection
569 :message "Received garbled message from backend"))))))
571 (defun read-null-bit-vector (socket count)
572 (let ((result (make-array count :element-type 'bit)))
573 (dotimes (offset (ceiling count 8))
574 (loop with byte = (read-byte socket)
575 for index from (* offset 8) below (min count (* (1+ offset) 8))
576 for weight downfrom 7
577 do (setf (aref result index) (ldb (byte 1 weight) byte))))
581 (defun read-field (socket type)
582 (let ((length (- (read-socket-value-int32 socket) 4)))
585 (read-integer-from-socket socket length))
587 (read-double-from-socket socket length))
589 (let ((result (make-string length)))
590 (read-socket-sequence result socket)
593 (uffi:def-constant +char-code-zero+ (char-code #\0))
594 (uffi:def-constant +char-code-minus+ (char-code #\-))
595 (uffi:def-constant +char-code-plus+ (char-code #\+))
596 (uffi:def-constant +char-code-period+ (char-code #\.))
597 (uffi:def-constant +char-code-lower-e+ (char-code #\e))
598 (uffi:def-constant +char-code-upper-e+ (char-code #\E))
600 (defun read-integer-from-socket (socket length)
601 (declare (fixnum length))
605 (first-char (read-byte socket))
607 (declare (fixnum first-char))
608 (decf length) ;; read first char
610 ((= first-char +char-code-minus+)
612 ((= first-char +char-code-plus+)
615 (setq val (- first-char +char-code-zero+))))
621 (- (read-byte socket) +char-code-zero+))))
626 (defmacro ascii-digit (int)
627 (let ((offset (gensym)))
628 `(let ((,offset (- ,int +char-code-zero+)))
629 (declare (fixnum ,int ,offset))
630 (if (and (>= ,offset 0)
635 (defun read-double-from-socket (socket length)
636 (declare (fixnum length))
637 (let ((before-decimal 0)
644 (char (read-byte socket)))
645 (declare (fixnum char exponent decimal-count))
646 (decf length) ;; already read first character
648 ((= char +char-code-minus+)
650 ((= char +char-code-plus+)
652 ((= char +char-code-period+)
655 (setq before-decimal (ascii-digit char))
656 (unless before-decimal
657 (error "Unexpected value"))))
661 (setq char (read-byte socket))
662 ;; (format t "~&len:~D, i:~D, char:~D, minusp:~A, decimalp:~A" length i char minusp decimalp)
663 (let ((weight (ascii-digit char)))
665 ((and weight (not decimalp)) ;; before decimal point
666 (setq before-decimal (+ weight (* 10 before-decimal))))
667 ((and weight decimalp) ;; after decimal point
668 (setq after-decimal (+ weight (* 10 after-decimal)))
669 (incf decimal-count))
670 ((and (= char +char-code-period+))
672 ((or (= char +char-code-lower-e+) ;; E is for exponent
673 (= char +char-code-upper-e+))
674 (setq exponent (read-integer-from-socket socket (- length i 1)))
675 (setq exponent (or exponent 0))
678 (break "Unexpected value"))
681 (setq result (* (+ (coerce before-decimal 'double-float)
683 (expt 10 (- decimal-count))))
691 (defun read-double-from-socket (socket length)
692 (let ((result (make-string length)))
693 (read-socket-sequence result socket)
694 (let ((*read-default-float-format* 'double-float))
695 (read-from-string result))))
697 (defun read-cursor-row (cursor types)
698 (let* ((connection (postgresql-cursor-connection cursor))
699 (socket (postgresql-connection-socket connection))
700 (fields (postgresql-cursor-fields cursor)))
701 (assert (postgresql-connection-open-p connection))
703 (let ((code (read-socket-value-int8 socket)))
705 (#.+ascii-row-message+
707 (loop with count = (length fields)
708 with null-vector = (read-null-bit-vector socket count)
710 for null-bit across null-vector
712 for null-p = (zerop null-bit)
717 (read-field socket (nth i types)))))
718 (#.+binary-row-message+
720 (#.+completed-response-message+
721 (return (values nil (read-socket-value-string socket))))
722 (#.+error-response-message+
723 (let ((message (read-socket-value-string socket)))
724 (error 'postgresql-error
725 :connection connection :message message)))
726 (#.+notice-response-message+
727 (let ((message (read-socket-value-string socket)))
728 (warn 'postgresql-warning
729 :connection connection :message message)))
730 (#.+notification-response-message+
731 (let ((pid (read-socket-value-int32 socket))
732 (message (read-socket-value-string socket)))
733 (when (= pid (postgresql-connection-pid connection))
734 (signal 'postgresql-notification :connection connection
737 (close-postgresql-connection connection)
738 (error 'postgresql-fatal-error :connection connection
739 :message "Received garbled message from backend")))))))
741 (defun map-into-indexed (result-seq func seq)
742 (dotimes (i (length seq))
744 (setf (elt result-seq i)
745 (funcall func (elt seq i) i)))
748 (defun copy-cursor-row (cursor sequence types)
749 (let* ((connection (postgresql-cursor-connection cursor))
750 (socket (postgresql-connection-socket connection))
751 (fields (postgresql-cursor-fields cursor)))
752 (assert (= (length fields) (length sequence)))
754 (let ((code (read-socket-value-int8 socket)))
756 (#.+ascii-row-message+
759 (let* ((count (length sequence))
760 (null-vector (read-null-bit-vector socket count)))
763 (if (zerop (elt null-vector i))
764 (setf (elt sequence i) nil)
765 (let ((value (read-field socket (nth i types))))
766 (setf (elt sequence i) value)))))
769 #'(lambda (null-bit i)
772 (read-field socket (nth i types))))
773 (read-null-bit-vector socket (length sequence)))))
774 (#.+binary-row-message+
776 (#.+completed-response-message+
777 (return (values nil (read-socket-value-string socket))))
778 (#.+error-response-message+
779 (let ((message (read-socket-value-string socket)))
780 (error 'postgresql-error
781 :connection connection :message message)))
782 (#.+notice-response-message+
783 (let ((message (read-socket-value-string socket)))
784 (warn 'postgresql-warning
785 :connection connection :message message)))
786 (#.+notification-response-message+
787 (let ((pid (read-socket-value-int32 socket))
788 (message (read-socket-value-string socket)))
789 (when (= pid (postgresql-connection-pid connection))
790 (signal 'postgresql-notification :connection connection
793 (close-postgresql-connection connection)
794 (error 'postgresql-fatal-error :connection connection
795 :message "Received garbled message from backend")))))))
797 (defun skip-cursor-row (cursor)
798 (let* ((connection (postgresql-cursor-connection cursor))
799 (socket (postgresql-connection-socket connection))
800 (fields (postgresql-cursor-fields cursor)))
802 (let ((code (read-socket-value-int8 socket)))
804 (#.+ascii-row-message+
805 (loop for null-bit across
806 (read-null-bit-vector socket (length fields))
808 (unless (zerop null-bit)
809 (let* ((length (read-socket-value-int32 socket)))
810 (loop repeat (- length 4) do (read-byte socket)))))
812 (#.+binary-row-message+
814 (#.+completed-response-message+
815 (return (values nil (read-socket-value-string socket))))
816 (#.+error-response-message+
817 (let ((message (read-socket-value-string socket)))
818 (error 'postgresql-error
819 :connection connection :message message)))
820 (#.+notice-response-message+
821 (let ((message (read-socket-value-string socket)))
822 (warn 'postgresql-warning
823 :connection connection :message message)))
824 (#.+notification-response-message+
825 (let ((pid (read-socket-value-int32 socket))
826 (message (read-socket-value-string socket)))
827 (when (= pid (postgresql-connection-pid connection))
828 (signal 'postgresql-notification :connection connection
831 (close-postgresql-connection connection)
832 (error 'postgresql-fatal-error :connection connection
833 :message "Received garbled message from backend")))))))
835 (defun run-query (connection query &optional (types nil))
836 (start-query-execution connection query)
837 (multiple-value-bind (status cursor)
838 (wait-for-query-results connection)
839 (assert (eq status :cursor))
840 (loop for row = (read-cursor-row cursor types)
844 (wait-for-query-results connection))))
847 (declaim (ext:maybe-inline read-byte write-byte))