Re: [yocto] Documentation around adding custom fetchers to layer
| Newsgroups | org.yoctoproject.lists.yocto |
|---|---|
| Message-ID | <[email protected]> |
Hi Alex,
Sure, I can share the source. It grabs secrets from our password manager (1Password) and injects them into the build system. We currently have two use cases for this:
1. We use AES keys for encrypting/decrypting our SWU payloads. Those key/iv pairs are held in our password manager, so we need to take one of them out at build-time and hand it to the swupdate bbclass to encrypt the payload. We do the same thing for a signing .pem key
2. We want to install an SSH key into our OS so that we can SSH into our devices. 1Password also acts as an SSH agent, so it makes life seamless when we store the keys in there.
My source is attached. The use case in a recipe would be:
SRC_URI = " \
op://MyVault/SSHKey/public%20key ;output=ssh_key.pub \
optemplate://aes_key.tpl;output=aes_key \
"
The output would be:
1. The content of the MyVault > SSHKey > public key dumped into a file called ssh_key.pub in the $WORKDIR
2. Depending on the template ( https://developer.1password.com/docs/cli/secrets-template-syntax/ ), ${WORKDIR}/aes_key would be filled in with referenced secrets
Someone at my last company wrote a custom fetcher to fetch conan packages, but I unfortunately wasn't super plugged in to what he was doing. I vaguely remember he did something with a bbclass and might have used something similar to the proposed do_fetch[prefuncs] method.
We currently have a hacky workaround right now where we have a *-init-build-env script that pulls the secrets from the 1Password vault(s) and put them in the correct place. But persisting secrets on-disk, unencrypted is non-ideal. My preference would be for us to only grab the secrets as we need them and then clean them up from build/ after they've been consumed.
Cheers,
Seth
op.py
(text/x-python, 4.5 KB)
import os
from pathlib import Path
import bb
from bb.fetch2 import FetchError, FetchMethod, runfetchcmd
class OpItem(FetchMethod):
"""Fetch method that runs `op read` on a 1Password CLI item path. URI should be in the format of:
op://<Vault>/<Item>/[<Section>/]<Field>
Note that all spaces in the string should be replaced with %20 so that bitbake can parse it properly.
"""
def supports(self, ud, d):
# Recognize URLs like: op://Vault/Item/[Section/]Field
return ud.type in ["op"]
def recommends_checksum(self, urldata):
return False
def supports_checksum(self, urldata):
return False
def urldata_init(self, ud, d):
# The 1Password path is given as the 'path' part of the URL.
# For example: op://Vault/Item/[Section/]Field
ud.path = ud.url.split("://", 1)[1].split(";")[0]
if not ud.path:
raise FetchError("Invalid op:// URI")
ud.path = f'op://{ud.path.replace("%20", " ")}'
# Optional query parameter: output=<filename>
ud.output = ud.parm.get("output") or os.path.basename(ud.path) + ".rendered"
# Store local destination path
ud.localfile = ud.output
def download(self, ud, d):
"""
Run `op read` to dump the contents of a field into a file.
"""
dl_dir = d.getVar("DL_DIR")
dest = os.path.join(dl_dir, ud.output)
os.environ["OP_SERVICE_ACCOUNT_TOKEN"] = d.getVar("OP_SERVICE_ACCOUNT_TOKEN")
cmd = f"/usr/local/bin/op read --out-file {dest} '{ud.path}'"
runfetchcmd(cmd, d)
if not os.path.exists(dest):
raise FetchError("op read did not produce an output file")
# The localfile is the file BitBake will checksum and stage.
ud.localpath = dest
def unpack(self, ud, destdir, d):
"""
Standard fetcher API: copy the injected file into the unpacked source dir.
"""
bb.utils.copyfile(ud.localpath, os.path.join(destdir, ud.output))
def checkstatus(self, fetch, urldata, d):
return True
class OpTemplate(FetchMethod):
"""Fetch method that runs `op inject` on a 1Password CLI injectable template."""
def supports(self, ud, d):
# Recognize URLs like: optemplate://path/to/template
return ud.type in ["optemplate"]
def recommends_checksum(self, urldata):
return False
def supports_checksum(self, urldata):
return False
def urldata_init(self, ud, d):
# The template file is given as the 'path' part of the URL.
# For example: optemplate:///absolute/path/to/template
ud.template = ud.url.split("://", 1)[1].split(";")[0]
if not ud.template:
raise FetchError("Missing template path in optemplate:// URL")
# Optional query parameter: output=<filename>
ud.output = ud.parm.get("output") or os.path.basename(ud.template) + ".rendered"
# Store local destination path
ud.localfile = ud.output
def download(self, ud, d):
"""
Run `op inject` to resolve secrets in the template and
produce the formatted output file.
"""
template_path = ""
if not os.path.exists(ud.template):
filespath = d.getVar("FILESPATH")
for location in filter(None, [Path(p) for p in filespath.split(":")]):
for glob_hit in location.glob(ud.template):
if glob_hit.exists():
template_path = location / ud.template
bb.note(f"Found template file at {template_path}")
break
if not template_path:
raise FetchError(f"Template file not found: {ud.template}")
dl_dir = d.getVar("DL_DIR")
dest = os.path.join(dl_dir, ud.output)
os.environ["OP_SERVICE_ACCOUNT_TOKEN"] = d.getVar("OP_SERVICE_ACCOUNT_TOKEN")
bb.note(f"Injecting 1Password template: {ud.template} -> {dest}")
cmd = f"/usr/local/bin/op inject -f -i {template_path} -o {dest}"
runfetchcmd(cmd, d)
if not os.path.exists(dest):
raise FetchError("op inject did not produce output file")
# The localfile is the file BitBake will checksum and stage.
ud.localpath = dest
def unpack(self, ud, destdir, d):
"""
Standard fetcher API: copy the injected file into the unpacked source dir.
"""
bb.utils.copyfile(ud.localpath, os.path.join(destdir, ud.output))
def checkstatus(self, fetch, urldata, d):
return True