Re: "nnweb survey: Please reply if you are using it"
James Thomas <[email protected]> Thu, 20 Feb 2025 14:42:43 +0530
| Newsgroups | gmane.emacs.gnus.general |
|---|---|
| Message-ID | <[email protected]> |
Björn Bidar writes: > James Thomas writes: > >> Andrew Cohen writes: >> >>> Using nnweb? >> >> Yes, but not with the defaults (which no longer work due to server-side >> changes); I wrote an extension which I use daily: >> >> https://codeberg.org/quotuva/nnweb-page/src/branch/main/nnweb-page.el > > That looks useful. My self I was wondering if something like this could > be adapted for mailman archives or the MHonArc archives which GNU uses. So here's a patch that changes nnweb defaults for help-gnu-emacs: WDYT? Please test if you can with: G w emacs-help RET unique string RET. (The article washing part wasn't worked on) --
nnweb-update.diff
(text/x-patch, 6.2 KB)
diff --git a/lisp/gnus/nnweb.el b/lisp/gnus/nnweb.el
index 57964f93437..02379a44600 100644
--- a/lisp/gnus/nnweb.el
+++ b/lisp/gnus/nnweb.el
@@ -40,20 +40,20 @@
(defvoo nnweb-directory (nnheader-concat gnus-directory "nnweb/")
"Where nnweb will save its files.")
-(defvoo nnweb-type 'google
+(defvoo nnweb-type 'emacs-help
"What search engine type is being used.
-Valid types include `google' and `dejanews'.")
+Valid types include `emacs-help' and `dejanews'.")
(defvar nnweb-type-definition
- '((google
- (id . "https://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source")
- (result . "https://groups.google.com/group/%s/msg/%s?dmode=source")
- (article . nnweb-google-wash-article)
- (reference . identity)
+ '((emacs-help
+ (id . "https://lists.gnu.org/archive/cgi-bin/namazu.cgi?idxname=help-gnu-emacs&sort=score&result=normal&max=20&submit=Search!&query=%%2Bmessage-id%%3A%%3C%s%%3E")
+ (result . "https://lists.gnu.org/archive/html/%s/%s.html")
+ (article . nnweb-gmane-wash-article)
+ (reference . nnweb-google-reference)
(map . nnweb-google-create-mapping)
(search . nnweb-google-search)
- (address . "https://groups.google.com/groups")
- (base . "https://groups.google.com")
+ (address . "https://lists.gnu.org/archive/cgi-bin/namazu.cgi")
+ (base . "https://lists.gnu.org")
(identifier . nnweb-google-identity))
;; FIXME: Make obsolete?
(dejanews ;; alias of google
@@ -315,7 +315,7 @@ nnweb-google-parse-1
(case-fold-search t)
(active (cadr (assoc nnweb-group nnweb-group-alist)))
Subject Date Newsgroups From
- map url mid)
+ map url mid link)
(unless active
(push (list nnweb-group (setq active (cons 1 0)))
nnweb-group-alist))
@@ -323,39 +323,34 @@ nnweb-google-parse-1
(goto-char (point-min))
(while
(re-search-forward
- "a +href=\"/group/\\([^>\"]+\\)/browse_thread/[^>]+#\\([0-9a-f]+\\)"
+ "<a +href=\"/archive/html/\\([^/]+\\)/\\([0-9-]+/msg[0-9]+\\)\\.html\">"
nil t)
- (setq Newsgroups (match-string-no-properties 1)
- ;; Note: Starting with Google Groups 2, `mid' is a Google-internal
- ;; ID, not a proper Message-ID.
+ (setq link (match-string-no-properties 0)
+ Newsgroups (match-string-no-properties 1)
+ ;; `mid' is not a proper Message-ID.
mid (match-string-no-properties 2)
url (format
(nnweb-definition 'result) Newsgroups mid))
- (narrow-to-region (search-forward ">" nil t)
+ (narrow-to-region (point)
(search-forward "</a>" nil t))
(mm-url-remove-markup)
(mm-url-decode-entities)
+ (replace-string "\n" "")
(setq Subject (buffer-string))
(goto-char (point-max))
(widen)
(narrow-to-region (point)
- (search-forward "</table" nil t))
+ (search-forward link nil t))
(mm-url-remove-markup)
(mm-url-decode-entities)
(goto-char (point-max))
(when
(re-search-backward
- "^\\(?:\\(\\w+\\) \\([0-9]+\\)\\|\\S-+\\)\\(?: \\([0-9]\\{4\\}\\)\\)? by ?\\(.*\\)"
+ "^Author: \\(.*\\)\nDate: \\(.*\\)"
nil t)
- (setq Date (if (match-string 1)
- (format "%s %s 00:00:00 %s"
- (match-string 1)
- (match-string 2)
- (or (match-string 3)
- (format-time-string "%Y")))
- (current-time-string)))
- (setq From (match-string 4)))
+ (setq Date (match-string-no-properties 2))
+ (setq From (match-string-no-properties 1)))
(widen)
(cl-incf i)
(unless (nnweb-get-hashtb url)
@@ -363,10 +358,8 @@ nnweb-google-parse-1
(list
(cl-incf (cdr active))
(make-full-mail-header
- (cdr active) (if Newsgroups
- (concat "(" Newsgroups ") " Subject)
- Subject)
- From Date (or Message-ID mid)
+ (cdr active) Subject
+ From Date (or Message-ID (concat Newsgroups "/" mid))
nil 0 0 url))
map)
(nnweb-set-hashtb (cadar map) (car map))))
@@ -384,10 +377,11 @@ nnweb-google-create-mapping
"Perform the search and create a number-to-url alist."
(with-current-buffer nnweb-buffer
(erase-buffer)
- (nnheader-message 7 "Searching google...")
+ (nnheader-message 7 "Searching...")
(when (funcall (nnweb-definition 'search) nnweb-search)
- (let ((more t)
- (i 0))
+ (let ((more 0)
+ (i 0)
+ link)
(while more
(setq nnweb-articles
(nconc nnweb-articles (nnweb-google-parse-1)))
@@ -395,18 +389,19 @@ nnweb-google-create-mapping
(goto-char (point-min))
(cl-incf i 100)
(if (or (not (re-search-forward
- "<a [^>]+href=\"\n?\\([^>\" \n\t]+\\)[^<]*<img[^>]+src=[^>]+next"
+ (format "<a +href=\"\\(/archive/cgi-bin/namazu.cgi?[^>]+&whence=%s\\)\"" (+ more 20))
nil t))
(>= i nnweb-max-hits))
(setq more nil)
;; Yup, there are more articles
- (setq more (concat (nnweb-definition 'base) (match-string 1)))
+ (setq more (+ more 20)
+ link (mm-url-decode-entities-string (match-string 1)))
(when more
(erase-buffer)
- (nnheader-message 7 "Searching google...(%d)" i)
- (mm-url-insert more))))
+ (nnheader-message 7 "Searching...(%d)" i)
+ (mm-url-insert (concat (nnweb-definition 'base) link)))))
;; Return the articles in the right order.
- (nnheader-message 7 "Searching google...done")
+ (nnheader-message 7 "Searching...done")
(setq nnweb-articles
(sort nnweb-articles #'car-less-than-car))))))
@@ -416,20 +411,18 @@ nnweb-google-search
(nnweb-definition 'address)
"?"
(mm-url-encode-www-form-urlencoded
- `(("q" . ,search)
- ("num" . ,(number-to-string
- (min 100 nnweb-max-hits)))
- ("hq" . "")
- ("hl" . "en")
- ("lr" . "")
- ("safe" . "off")
- ("sites" . "groups")
- ("filter" . "0")))))
+ `(("idxname" . "help-gnu-emacs")
+ ("sort" . "score")
+ ("result" . "normal")
+ ("max" . ,(number-to-string
+ (min 20 nnweb-max-hits)))
+ ("submit" . "Search!")
+ ("query" . ,search)))))
t)
(defun nnweb-google-identity (url)
"Return a unique identifier based on URL."
- (if (string-match "selm=\\([^ &>]+\\)" url)
+ (if (string-match "archive/html/\\(.*\\)\\.html" url)
(match-string 1 url)
url))