Re: fast serialization of huge bit vectors?
Shiro Kawai <[email protected]> Wed, 6 May 2026 20:33:18 -1000
| Newsgroups | gmane.lisp.scheme.gauche |
|---|---|
| Message-ID | <CALN0JNGxxLP9b9vWcfuPmdV4K3Z7xM3yyiqt5O7Wd-Wf4k2Y=Q@mail.gmail.com> |
--===============2643121359488697841== Content-Type: multipart/alternative; boundary="00000000000060e3a20651347563" --00000000000060e3a20651347563 Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: quoted-printable bitvector->uvector/shared and uvector->bitvector/shared are added. https://github.com/shirok/Gauche/commit/57303f6feadfdb592e1c4e0f25f76b7c47e= db8c4 On Mon, May 4, 2026 at 11:14=E2=80=AFPM Jens Thiele <[email protected]> wrote: > just in case somebody wants a quick hack for now. Something like this > should work: > > (define-module bitvector-io > (use gauche.bitvector) > (use binary.io) > (use runtime-compile) > (export read-bitvector > write-bitvector)) > > (select-module bitvector-io) > > (compile-and-load > `((inline-stub > (define-cproc read-bitvector-bits (v::<bitvector> p::<input-port>) > (let* ((bytes::ScmSize (* (+ (/ (SCM_BITVECTOR_SIZE v) SCM_WORD_BIT= S) > (!=3D (% (SCM_BITVECTOR_SIZE v) > SCM_WORD_BITS) 0)) > (/ SCM_WORD_BITS 8))) > (r::ScmSize (Scm_Getz (cast (char *) (SCM_BITVECTOR_BITS v)) > bytes p))) > (return (SCM_MAKE_BOOL (=3D=3D bytes r))))) > (define-cproc write-bitvector-bits (v::<bitvector> p::<output-port>) > (Scm_Putz (cast (char *) (SCM_BITVECTOR_BITS v)) > (* (+ (/ (SCM_BITVECTOR_SIZE v) SCM_WORD_BITS) > (!=3D (% (SCM_BITVECTOR_SIZE v) SCM_WORD_BITS) 0)) > (/ SCM_WORD_BITS 8)) > p) > (return SCM_UNDEFINED)))) > '(read-bitvector-bits write-bitvector-bits)) > > (define (read-bitvector) > (let1 len (read-u64) > (rlet1 r (make-bitvector len) > (when (not (read-bitvector-bits r (current-input-port))) > (error "read-bitvector: read failure"))))) > > (define (write-bitvector v) > (write-u64 (bitvector-length v)) > (write-bitvector-bits v (current-output-port))) > > ;; (let1 v > #*11111111111111111111111111111111111111111111111111111111111111111 > (with-output-to-file "/tmp/v" (lambda() (write-bitvector v))) (equal? v > (with-input-from-file "/tmp/v" (lambda() (read-bitvector))))) > > > _______________________________________________ > Gauche-devel mailing list > [email protected] > https://lists.sourceforge.net/lists/listinfo/gauche-devel > --00000000000060e3a20651347563 Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: quoted-printable <div dir=3D"ltr"><div class=3D"gmail_default" style=3D"font-size:large">bit= vector->uvector/shared and uvector->bitvector/shared are added.=C2=A0= =C2=A0</div><div class=3D"gmail_default" style=3D"font-size:large"><a href= =3D"https://github.com/shirok/Gauche/commit/57303f6feadfdb592e1c4e0f25f76b7= c47edb8c4">https://github.com/shirok/Gauche/commit/57303f6feadfdb592e1c4e0f= 25f76b7c47edb8c4</a></div></div><br><div class=3D"gmail_quote gmail_quote_c= ontainer"><div dir=3D"ltr" class=3D"gmail_attr">On Mon, May 4, 2026 at 11:1= 4=E2=80=AFPM Jens Thiele <<a href=3D"mailto:[email protected]">karme@karme.= de</a>> wrote:<br></div><blockquote class=3D"gmail_quote" style=3D"margi= n:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex= ">just in case somebody wants a quick hack for now. Something like this<br> should work:<br> <br> (define-module bitvector-io<br> =C2=A0 (use gauche.bitvector)<br> =C2=A0 (use <a href=3D"http://binary.io" rel=3D"noreferrer" target=3D"_blan= k">binary.io</a>)<br> =C2=A0 (use runtime-compile)<br> =C2=A0 (export read-bitvector<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 write-bitvector))<br> <br> (select-module bitvector-io)<br> <br> (compile-and-load<br> =C2=A0`((inline-stub<br> =C2=A0 =C2=A0 (define-cproc read-bitvector-bits (v::<bitvector> p::&l= t;input-port>)<br> =C2=A0 =C2=A0 =C2=A0 (let* ((bytes::ScmSize (* (+ (/ (SCM_BITVECTOR_SIZE v)= SCM_WORD_BITS)<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0(!=3D (% (SCM_BITVECTOR= _SIZE v) SCM_WORD_BITS) 0))<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 (/ SCM_WORD_BITS 8)))<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0(r::ScmSize (Scm_Getz (cast= (char *) (SCM_BITVECTOR_BITS v)) bytes p)))<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 (return (SCM_MAKE_BOOL (=3D=3D bytes r)))))<br> =C2=A0 =C2=A0 (define-cproc write-bitvector-bits (v::<bitvector> p::&= lt;output-port>)<br> =C2=A0 =C2=A0 =C2=A0 (Scm_Putz (cast (char *) (SCM_BITVECTOR_BITS v))<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 (* (+ (/ (SCM_BITVE= CTOR_SIZE v) SCM_WORD_BITS)<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 (!=3D (% (SCM_BITVECTOR_SIZE v) SCM_WORD_BITS) 0))<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0(/ SCM= _WORD_BITS 8))<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 p)<br> =C2=A0 =C2=A0 =C2=A0 (return SCM_UNDEFINED))))<br> =C2=A0'(read-bitvector-bits write-bitvector-bits))<br> <br> (define (read-bitvector)<br> =C2=A0 (let1 len (read-u64)<br> =C2=A0 =C2=A0 (rlet1 r (make-bitvector len)<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0(when (not (read-bitvector-bits r = (current-input-port)))<br> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0(error "read-bitvector= : read failure")))))<br> <br> (define (write-bitvector v)<br> =C2=A0 (write-u64 (bitvector-length v))<br> =C2=A0 (write-bitvector-bits v (current-output-port)))<br> <br> ;; (let1 v #*11111111111111111111111111111111111111111111111111111111111111= 111 (with-output-to-file "/tmp/v" (lambda() (write-bitvector v)))= (equal? v (with-input-from-file "/tmp/v" (lambda() (read-bitvect= or)))))<br> <br> <br> _______________________________________________<br> Gauche-devel mailing list<br> <a href=3D"mailto:[email protected]" target=3D"_blank">Gau= [email protected]</a><br> <a href=3D"https://lists.sourceforge.net/lists/listinfo/gauche-devel" rel= =3D"noreferrer" target=3D"_blank">https://lists.sourceforge.net/lists/listi= nfo/gauche-devel</a><br> </blockquote></div> --00000000000060e3a20651347563-- --===============2643121359488697841== Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Disposition: inline --===============2643121359488697841== Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Disposition: inline _______________________________________________ Gauche-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/gauche-devel --===============2643121359488697841==--