思源笔记用数据库做网页信息收集的 demo

流程及效果

在网页上添加一个按钮，点击按钮时，通过配置匹配不同网页获取不同网页信息，插入配置的数据库。

​

​

使用场景示例

• 书签收集
• 保存网页上一些需要的信息

demo 使用

通过油猴插件新增脚本，脚本代码：

// ==UserScript==
// @name         思源网页信息采集器
// @namespace    https://leay.net/
// @version      2.4
// @description  获取网页不同信息并根据网址规则插入到不同的思源笔记数据库
// @author       hqweay
// @match        *://*/*
// @grant        GM_setClipboard
// @grant        GM_xmlhttpRequest
// @grant        GM_getValue
// @grant        GM_setValue
// @grant        GM_registerMenuCommand
// @connect      localhost
// ==/UserScript==

(function () {
  "use strict";

  // 序列化配置（保存时调用）
  function serializeConfig(config) {
    return JSON.stringify(config, (key, value) => {
      if (value instanceof RegExp) {
        return {
          __regex__: true, // 自定义标记
          source: value.source, // 正则主体
          flags: value.flags, // 修饰符 (i,g,m等)
        };
      }
      return value;
    });
  }

  // 反序列化配置（加载时调用）
  function deserializeConfig(jsonStr) {
    return JSON.parse(jsonStr, (key, value) => {
      if (value?.__regex__) {
        return new RegExp(value.source, value.flags);
      }
      return value;
    });
  }

  // 保存配置
  function saveConfig() {
    GM_setValue("siyuan-config", serializeConfig(config));
  }

  // 加载配置
  function loadConfig() {
    const saved = GM_getValue("siyuan-config");
    return saved ? deserializeConfig(saved) : defaultConfig;
  }

  // 默认配置
  const defaultConfig = {
    baseURL: "http://localhost:6806",
    token: "",
    rules: [
      {
        name: "默认规则",
        urlPattern: /.*/,
        databaseID: "20250908210738-aaiw4zu",
        columns: {
          title: { name: "标题", type: "block" },
          url: { name: "网址", type: "url" },
          thumbnail: { name: "封面", type: "mAsset" },
          keywords: { name: "标签", type: "mSelect" },
        },
        extractMethod: "meta",
      },
      {
        name: "doubanBook元数据",
        urlPattern:
          /^https:\/\/book\.douban\.com\/subject\/(\d+)(\/|\/?\?.*)?$/i,
        databaseID: "20250908210738-6kaxs8n",
        columns: {
          title: { name: "名称", type: "block" },
          author: { name: "作者", type: "mSelect" },
          cover: { name: "封面", type: "mAsset" },
          isbn: { name: "ISBN", type: "text" },
          url: { name: "链接", type: "url" },
        },
        extractMethod: "doubanBook",
      },
    ],
  };

  // 加载配置
  let config = loadConfig();

  // 初始化正则表达式
  function initRegexPatterns() {
    config.rules.forEach((rule) => {
      if (typeof rule.urlPattern === "string") {
        try {
          const pattern = rule.urlPattern;
          const flags = pattern.endsWith("/i") ? "i" : "";
          const cleanPattern = flags ? pattern.slice(0, -2) : pattern;
          rule.compiledPattern = new RegExp(cleanPattern, flags);
        } catch (e) {
          console.error("无效的正则表达式:", rule.urlPattern, e);
          rule.compiledPattern = /.*/;
        }
      } else if (rule.urlPattern instanceof RegExp) {
        rule.compiledPattern = rule.urlPattern;
      } else {
        rule.compiledPattern = /.*/;
      }
    });
  }
  initRegexPatterns();

  // 配置管理界面
  function showConfigEditor() {
    const editor = document.createElement("div");
    editor.style.position = "fixed";
    editor.style.top = "50%";
    editor.style.left = "50%";
    editor.style.transform = "translate(-50%, -50%)";
    editor.style.backgroundColor = "white";
    editor.style.padding = "20px";
    editor.style.borderRadius = "5px";
    editor.style.boxShadow = "0 0 10px rgba(0,0,0,0.3)";
    editor.style.zIndex = "10000";
    editor.style.width = "80%";
    editor.style.maxWidth = "600px";
    editor.style.maxHeight = "80vh";
    editor.style.overflow = "auto";

    editor.innerHTML = `
					<h2 style="margin-top: 0;">思源笔记采集器配置</h2>
					<div>
							<label>API地址: <input type="text" id="config-baseURL" value="${config.baseURL}" style="width: 100%;"></label>
					</div>
					<div>
							<label>Token: <input type="text" id="config-token" value="${config.token}" style="width: 100%;"></label>
					</div>
					<h3>采集规则</h3>
					<div id="rules-container"></div>
					<div style="margin-top: 10px; display: flex; gap: 10px;">
							<button id="add-rule">添加规则</button>
							<button id="restore-defaults" style="background-color: #ff9500;">恢复默认配置</button>
					</div>
					<div style="margin-top: 20px; display: flex; justify-content: space-between;">
							<button id="save-config">保存配置</button>
							<button id="cancel-config">取消</button>
					</div>
			`;

    const rulesContainer = editor.querySelector("#rules-container");

    // 渲染现有规则
    config.rules.forEach((rule, index) => {
      renderRule(rule, index);
    });

    function renderRule(rule, index) {
      const ruleDiv = document.createElement("div");
      ruleDiv.style.border = "1px solid #ddd";
      ruleDiv.style.padding = "10px";
      ruleDiv.style.marginBottom = "10px";
      ruleDiv.style.position = "relative";

      console.log(rule.urlPattern instanceof RegExp);

      const displayUrlPattern =
        rule.urlPattern instanceof RegExp
          ? `${rule.urlPattern.source}${rule.urlPattern.ignoreCase ? "/i" : ""}`
          : rule.urlPattern;

      ruleDiv.innerHTML = `
							<h4 style="margin-top: 0;">规则 ${index + 1}</h4>
							<div>
									<label>规则名称: <input type="text" class="rule-name" value="${
                    rule.name
                  }" style="width: 100%;"></label>
							</div>
							<div>
									<label>URL匹配规则(正则): <input type="text" class="rule-urlPattern" value="${displayUrlPattern}" style="width: 100%;"></label>
							</div>
							<div>
									<label>数据库ID: <input type="text" class="rule-databaseID" value="${
                    rule.databaseID
                  }" style="width: 100%;"></label>
							</div>
							<div>
									<label>提取方法:
											<select class="rule-extractMethod" style="width: 100%;">
													<option value="meta" ${
                            rule.extractMethod === "meta" ? "selected" : ""
                          }>Meta标签</option>

													<option value="doubanBook" ${
                            rule.extractMethod === "doubanBook"
                              ? "selected"
                              : ""
                          }>doubanBook元数据</option>
											</select>
									</label>
							</div>
							<h5>列映射</h5>
							<div class="columns-container"></div>
							<button class="add-column" style="margin-top: 5px;">添加列</button>
							<button class="remove-rule" style="position: absolute; top: 10px; right: 10px; background-color: #ff4757; color: white;">删除</button>
					`;

      const columnsContainer = ruleDiv.querySelector(".columns-container");

      Object.entries(rule.columns || {}).forEach(([field, col]) => {
        renderColumn(field, col.name, col.type);
      });

      function renderColumn(field = "", colName = "", colType = "text") {
        const columnDiv = document.createElement("div");
        columnDiv.style.display = "grid";
        columnDiv.style.gridTemplateColumns = "1fr 1fr 1fr 50px";
        columnDiv.style.gap = "5px";
        columnDiv.style.marginBottom = "5px";
        columnDiv.style.alignItems = "center";

        columnDiv.innerHTML = `
									<input type="text" class="column-field" value="${field}" placeholder="元数据字段">
									<select class="column-type">
											<option value="text" ${colType === "text" ? "selected" : ""}>文本</option>
											<option value="block" ${colType === "block" ? "selected" : ""}>块</option>
											<option value="mSelect" ${colType === "mSelect" ? "selected" : ""}>多选</option>
											<option value="mAsset" ${colType === "mAsset" ? "selected" : ""}>资源</option>
											<option value="number" ${colType === "number" ? "selected" : ""}>数字</option>
											<option value="url" ${colType === "url" ? "selected" : ""}>链接</option>
									</select>
									<input type="text" class="column-name" value="${colName}" placeholder="数据库列名">
									<button class="remove-column" style="background-color: #ff4757; color: white;">×</button>
							`;

        columnsContainer.appendChild(columnDiv);
      }

      ruleDiv.querySelector(".add-column").addEventListener("click", () => {
        renderColumn("", "", "text");
      });

      ruleDiv.querySelector(".remove-rule").addEventListener("click", () => {
        if (config.rules.length <= 1) {
          alert("至少需要保留一条规则");
          return;
        }
        ruleDiv.remove();
      });

      rulesContainer.appendChild(ruleDiv);
    }

    editor.querySelector("#add-rule").addEventListener("click", () => {
      renderRule(
        {
          name: "新规则",
          urlPattern: ".*",
          databaseID: "",
          extractMethod: "meta",
          columns: {},
        },
        config.rules.length
      );
    });

    // 恢复默认配置按钮
    editor.querySelector("#restore-defaults").addEventListener("click", () => {
      if (confirm("确定要恢复默认配置吗？这将清除所有自定义规则。")) {
        config = defaultConfig; //JSON.parse(JSON.stringify(defaultConfig));
        rulesContainer.innerHTML = "";
        config.rules.forEach((rule, index) => {
          renderRule(rule, index);
        });
        showNotification("已恢复默认配置");
      }
    });

    editor.querySelector("#save-config").addEventListener("click", () => {
      const newConfig = {
        baseURL: editor.querySelector("#config-baseURL").value,
        token: editor.querySelector("#config-token").value,
        rules: [],
      };

      editor.querySelectorAll("#rules-container > div").forEach((ruleDiv) => {
        const patternInput = ruleDiv.querySelector(".rule-urlPattern").value;
        let urlPattern;

        try {
          // 处理带/i标志的情况
          if (patternInput.endsWith("/i")) {
            urlPattern = new RegExp(patternInput.slice(0, -2), "i");
          } else {
            urlPattern = new RegExp(patternInput);
          }
        } catch (e) {
          console.error("无效的正则表达式:", patternInput);
          urlPattern = /.*/; // 默认匹配所有
        }
        const rule = {
          name: ruleDiv.querySelector(".rule-name").value,
          urlPattern: urlPattern,
          databaseID: ruleDiv.querySelector(".rule-databaseID").value,
          extractMethod: ruleDiv.querySelector(".rule-extractMethod").value,
          columns: {},
        };

        ruleDiv
          .querySelectorAll(".columns-container > div")
          .forEach((colDiv) => {
            const field = colDiv.querySelector(".column-field").value;
            const colName = colDiv.querySelector(".column-name").value;
            const colType = colDiv.querySelector(".column-type").value;

            if (field && colName) {
              rule.columns[field] = {
                name: colName,
                type: colType,
              };
            }
          });

        newConfig.rules.push(rule);
      });

      config = newConfig;
      // GM_setValue("siyuan-config", JSON.stringify(config));
      saveConfig();
      initRegexPatterns();
      editor.remove();
      showNotification("配置已保存");
    });

    editor.querySelector("#cancel-config").addEventListener("click", () => {
      editor.remove();
    });

    document.body.appendChild(editor);
  }

  // 注册配置菜单命令
  GM_registerMenuCommand("配置思源笔记采集器", showConfigEditor);

  // 创建UI元素
  const style = document.createElement("style");
  style.textContent = `
			#metaInfoButton {
					position: fixed;
					bottom: 20px;
					right: 20px;
					padding: 10px 15px;
					background-color: #ff4757;
					color: white;
					border: none;
					border-radius: 5px;
					cursor: pointer;
					font-weight: bold;
					z-index: 9999;
			}
			#metaInfoButton:hover {
					background-color: #ff6b81;
			}
			#metaInfoContainer {
					position: fixed;
					bottom: 70px;
					right: 20px;
					width: 300px;
					max-height: 400px;
					overflow-y: auto;
					background-color: white;
					border: 1px solid #ddd;
					border-radius: 5px;
					padding: 15px;
					box-shadow: 0 0 10px rgba(0,0,0,0.1);
					z-index: 9998;
					display: none;
			}
			.meta-item {
					margin-bottom: 10px;
					padding-bottom: 10px;
					border-bottom: 1px solid #eee;
			}
			.meta-name {
					font-weight: bold;
					color: #ff4757;
			}
			.copy-notification {
					position: fixed;
					bottom: 70px;
					right: 20px;
					background-color: #4CAF50;
					color: white;
					padding: 10px 15px;
					border-radius: 5px;
					z-index: 10000;
					display: none;
			}
	`;
  document.head.appendChild(style);

  const button = document.createElement("button");
  button.id = "metaInfoButton";
  button.textContent = "采集到思源笔记";
  document.body.appendChild(button);

  const container = document.createElement("div");
  container.id = "metaInfoContainer";
  document.body.appendChild(container);

  const notification = document.createElement("div");
  notification.className = "copy-notification";
  document.body.appendChild(notification);

  function showNotification(message, isError = false) {
    notification.textContent = message;
    notification.style.backgroundColor = isError ? "#ff4757" : "#4CAF50";
    notification.style.display = "block";
    setTimeout(() => {
      notification.style.display = "none";
    }, 3000);
  }

  // 辅助函数：通过文本内容查找元素
  function findElementByText(text) {
    const spans = document.querySelectorAll("#info span.pl");
    for (const span of spans) {
      if (span.textContent.includes(text)) {
        return span;
      }
    }
    return null;
  }
  // 元数据提取器
  const extractors = {
    // 默认meta标签提取
    meta: function () {
      const url = cleanUrl(window.location.href);
      return {
        title:
          document
            .querySelector("meta[property='og:title']")
            ?.getAttribute("content")
            ?.trim() ||
          document.querySelector("title")?.textContent?.trim() ||
          "",
        url: url,
        thumbnail:
          document
            .querySelector("meta[property='og:image']")
            ?.getAttribute("content") ||
          document.querySelector("link[rel='icon']")?.getAttribute("href") ||
          "",
        keywords:
          document
            .querySelector("meta[name='keywords']")
            ?.getAttribute("content") ||
          document
            .querySelector("meta[name='Keywords']")
            ?.getAttribute("content") ||
          "",
      };
    },
    doubanBook: function () {
      const meta = extractors.meta(); // 先获取基础meta信息
      const url = meta.url;

      // 提取书籍ID
      const match = url.match(/\/subject\/(\d+)/);
      meta.id = (match && match[1]) || meta.id;

      // 提取书名
      const titleElement = document.querySelector("h1 span");
      meta.title = (titleElement && titleElement.textContent.trim()) || "";

      const coverElement = document.querySelector("#mainpic img");
      meta.cover =
        (coverElement && coverElement.getAttribute("src").trim()) || "";

      // 提取作者
      const authorElement = document.querySelector("#info span:first-child a");
      meta.author = (authorElement && authorElement.textContent.trim()) || "";

      // 提取出版社
      const publisherElement = findElementByText("出版社");
      meta.publisher =
        (publisherElement &&
          publisherElement.nextElementSibling &&
          publisherElement.nextElementSibling.textContent.trim()) ||
        "";

      // 提取出品方
      const producerElement = findElementByText("出品方");
      meta.producer =
        (producerElement &&
          producerElement.nextElementSibling &&
          producerElement.nextElementSibling.textContent.trim()) ||
        "";

      // 提取副标题
      const subtitleElement = findElementByText("副标题");
      meta.subtitle =
        (subtitleElement &&
          subtitleElement.nextSibling &&
          subtitleElement.nextSibling.textContent.trim()) ||
        "";

      // 提取出版年
      const publishDateElement = findElementByText("出版年");
      meta.publishDate =
        (publishDateElement &&
          publishDateElement.nextSibling &&
          publishDateElement.nextSibling.textContent.trim()) ||
        "";

      // 提取页数
      const pagesElement = findElementByText("页数");
      meta.pages =
        (pagesElement &&
          pagesElement.nextSibling &&
          pagesElement.nextSibling.textContent.trim()) ||
        "";

      // 提取定价
      const priceElement = findElementByText("定价");
      meta.price =
        (priceElement &&
          priceElement.nextSibling &&
          priceElement.nextSibling.textContent.trim()) ||
        "";

      // 提取装帧
      const bindingElement = findElementByText("装帧");
      meta.binding =
        (bindingElement &&
          bindingElement.nextSibling &&
          bindingElement.nextSibling.textContent.trim()) ||
        "";

      // 提取ISBN
      const isbnElement = findElementByText("ISBN");
      meta.isbn =
        (isbnElement &&
          isbnElement.nextSibling &&
          isbnElement.nextSibling.textContent.trim()) ||
        "";

      // 确保URL是干净的（去除查询参数）
      meta.url = `https://book.douban.com/subject/${meta.id}/`;

      return meta;
    },
  };

  // 提取元数据（根据匹配的规则选择提取方法）
  function extractMetadata(rule) {
    const extractMethod = rule.extractMethod || "meta";
    const extractor = extractors[extractMethod] || extractors.meta;
    return extractor();
  }

  function cleanUrl(urlString) {
    const url = new URL(urlString);
    const paramsToRemove = [
      "utm_source",
      "utm_medium",
      "utm_campaign",
      "utm_term",
      "utm_content",
      "fbclid",
      "gclid",
      "yclid",
      "msclkid",
      "icid",
      "mc_cid",
      "mc_eid",
      "_ga",
      "si",
      "igshid",
      "feature",
      "sharing",
      "app",
      "ref",
      "nr",
      "ncid",
      "cmpid",
      "ito",
      "ved",
      "ei",
      "s",
      "cvid",
      "form",
    ];
    paramsToRemove.forEach((param) => url.searchParams.delete(param));
    return url.toString();
  }

  function callSiyuanAPI(endpoint, data = {}, method = "POST") {
    console.log(data);
    return new Promise((resolve, reject) => {
      GM_xmlhttpRequest({
        method: method,
        url: `${config.baseURL}${endpoint}`,
        headers: {
          "Content-Type": "application/json",
          Authorization: `Token ${config.token}`,
        },
        data: JSON.stringify(data),
        onload: (response) => {
          try {
            const result = JSON.parse(response.responseText);
            if (result.code === 0) {
              resolve(result.data);
            } else {
              reject(new Error(result.msg || "API调用失败"));
            }
          } catch (e) {
            reject(e);
          }
        },
        onerror: (error) => {
          reject(error);
        },
      });
    });
  }

  async function getDatabaseColumns(avId) {
    try {
      const keys = await callSiyuanAPI("/api/av/getAttributeViewKeysByAvID", {
        avID: avId,
      });
      return keys || [];
    } catch (error) {
      console.error("获取数据库列信息失败:", error);
      return [];
    }
  }

  function formatValue(value, type) {
    switch (type) {
      case "text":
        return { text: { content: value || "" } };
      case "block":
        return { block: { content: value || "" } };
      case "mSelect":
        return {
          mSelect: value
            ? value.split(",").map((item) => ({ content: item.trim() }))
            : [],
        };
      case "mAsset":
        return { mAsset: value ? [{ content: value }] : [] };
      case "number":
        return { number: { content: Number(value) || 0 } };
      case "url":
        return { url: { content: value || "" } };
      default:
        return { text: { content: value || "" } };
    }
  }

  async function insertToSiyuanDatabase(metadata, rule) {
    try {
      const avId = rule.databaseID;
      const columns = await getDatabaseColumns(avId);

      if (columns.length === 0) throw new Error("数据库列信息获取失败");

      const values = [];
      const columnMap = {};
      columns.forEach(
        (col) => (columnMap[col.name] = { id: col.id, type: col.type })
      );

      // 主键列（第一列）
      const pkKeyID = columns[0].id;
      const pkColumnName = columns[0].name; // 获取主键列名

      values.push({
        keyID: pkKeyID,
        ...formatValue(metadata.title || "无标题", columns[0].type),
      });

      console.log(rule);
      console.log(columnMap);

      // 其他列（排除主键列）
      Object.entries(rule.columns).forEach(([field, col]) => {
        // 检查不是主键列且字段存在
        if (
          col.name !== pkColumnName &&
          columnMap[col.name] &&
          metadata[field] !== undefined
        ) {
          values.push({
            keyID: columnMap[col.name].id,
            ...formatValue(metadata[field], col.type),
          });
        }
      });

      const input = {
        avID: avId,
        blocksValues: [values],
      };

      console.log(input);

      await callSiyuanAPI(
        "/api/av/appendAttributeViewDetachedBlocksWithValues",
        input
      );
    } catch (error) {
      console.error("插入数据失败:", error);
      throw error;
    }
  }

  function getMatchingRule(url) {
    console.log("match");
    console.log(url);
    console.log(config.rules);
    for (let i = config.rules.length - 1; i >= 0; i--) {
      const rule = config.rules[i];
      if (rule.compiledPattern.test(url)) return rule;
    }
    return config.rules[0];
  }

  button.addEventListener("click", async function () {
    try {
      const currentUrl = cleanUrl(window.location.href);
      const rule = getMatchingRule(currentUrl);
      const metadata = extractMetadata(rule); // 使用规则指定的提取方法

      container.innerHTML = `
							<div style="margin-bottom: 10px; font-weight: bold;">将保存到: ${rule.name}</div>
							<div style="margin-bottom: 10px; color: #666;">数据库ID: ${rule.databaseID}</div>
					`;
      console.log("metadata");
      console.log(rule);
      console.log(metadata);

      Object.entries(metadata).forEach(([field, value]) => {
        if (rule.columns[field]) {
          const item = document.createElement("div");
          item.className = "meta-item";
          item.innerHTML = `
											<div class="meta-name">${rule.columns[field].name}</div>
											<div class="meta-content">${value}</div>
									`;
          container.appendChild(item);
        }
      });
      container.style.display = "block";

      await insertToSiyuanDatabase(metadata, rule);
      showNotification(`网页信息已保存到【${rule.name}】数据库`);
      GM_setClipboard(JSON.stringify(metadata, null, 2), "text");

      setTimeout(() => {
        container.style.display = "none";
      }, 2000);
    } catch (error) {
      console.error("保存失败:", error);
      showNotification(`保存失败: ${error.message}`, true);
    }
  });
})();

启用脚本后，打开脚本配置

​

默认情况下，傻瓜化使用方式：

• 配置 Token
• 配置默认规则的数据库 ID
• 配置 doubanBook 元数据的数据库 ID

数据库模板下载：

网页元数据抓取到数据库测试模板。sy.zip​

然后可以随便打开一个网页或者豆瓣书籍网页测试：https://book.douban.com/subject/37345943/

说明

抛砖引玉分享下，实现了一个小但展现了可能性的流程。

• 保存书签
• 保存豆瓣书籍信息

可以扩展的地方：目前通过正则匹配网页与获取网页信息的逻辑是 1 对 1 的，应该能支持一个正则下的网页支持多种网页信息获取的逻辑。然后在「采集」时也能支持选择哪一种。

还可以有哪些场景？

其中数据库相关操作参考：[js] 添加文档 / 块到指定数据库（支持添加任意字段）