Sync third-party skill plugins from upstream repos

Constraint: Maintainers need repeatable ingestion for external skill repositories with different layouts.\nRejected: Git submodules as the primary mechanism | Codex marketplace installs do not reliably recurse external submodules, and adapters still need layout conversion.\nConfidence: medium\nScope-risk: moderate\nDirective: Add new upstreams through config/external-sources.json and adapter logic rather than hand-copying plugin folders.\nTested: scripts/sync_external_plugins.ps1 -Proxy http://127.0.0.1:18085; scripts/sync_external_plugins.ps1 -Only ui-ux-pro-max -Proxy http://127.0.0.1:18085; python scripts/validate_marketplace.py; validate_plugin.py for superpowers and ui-ux-pro-max; codex plugin add superpowers@eapil-skill-market; codex plugin add ui-ux-pro-max@eapil-skill-market; verified ui-ux-pro-max data/scripts directories in cache.\nNot-tested: Scheduled .gitea workflow execution on git.playones runner.
This commit is contained in:
AreChen
2026-06-10 15:34:28 +08:00
parent d951b7cf05
commit 843b4e23c4
100 changed files with 20487 additions and 324 deletions
+429
View File
@@ -0,0 +1,429 @@
param(
[string]$ConfigPath = "config/external-sources.json",
[string]$WorkDir = "",
[string]$Only = "",
[string]$Proxy = "",
[switch]$KeepTemp
)
$ErrorActionPreference = "Stop"
if ($Proxy) {
$env:HTTP_PROXY = $Proxy
$env:HTTPS_PROXY = $Proxy
$env:ALL_PROXY = $Proxy
}
function Invoke-CheckedGit {
param([string[]]$GitArgs)
& git @GitArgs
if ($LASTEXITCODE -ne 0) {
throw "git $($GitArgs -join ' ') failed with exit code $LASTEXITCODE"
}
}
function Resolve-RepoRoot {
$root = git rev-parse --show-toplevel
if (-not $root) {
throw "Run this script from inside a git repository."
}
return (Resolve-Path $root.Trim()).Path
}
function Assert-ChildPath {
param(
[string]$Parent,
[string]$Child
)
$parentPath = (Resolve-Path $Parent).Path
$childPath = $Child
if (Test-Path -LiteralPath $Child) {
$childPath = (Resolve-Path $Child).Path
} else {
$childPath = [System.IO.Path]::GetFullPath($Child)
}
if (-not $childPath.StartsWith($parentPath, [System.StringComparison]::OrdinalIgnoreCase)) {
throw "Refusing to write outside $parentPath`: $childPath"
}
}
function Remove-PathIfExists {
param([string]$Path, [string]$Within)
if (Test-Path -LiteralPath $Path) {
Assert-ChildPath -Parent $Within -Child $Path
Remove-Item -LiteralPath $Path -Recurse -Force
}
}
function Copy-PathIfExists {
param(
[string]$Source,
[string]$Destination,
[string]$Within
)
if (Test-Path -LiteralPath $Source) {
Assert-ChildPath -Parent $Within -Child $Destination
Copy-Item -LiteralPath $Source -Destination $Destination -Recurse -Force
}
}
function Read-JsonFile {
param([string]$Path)
return Get-Content -Raw -LiteralPath $Path | ConvertFrom-Json
}
function Write-JsonFile {
param([string]$Path, [object]$Value)
$json = $Value | ConvertTo-Json -Depth 20
$utf8NoBom = New-Object System.Text.UTF8Encoding($false)
[System.IO.File]::WriteAllText($Path, "$json`n", $utf8NoBom)
}
function ConvertTo-Hashtable {
param([object]$InputObject)
if ($null -eq $InputObject) {
return $null
}
if ($InputObject -is [System.Collections.IEnumerable] -and -not ($InputObject -is [string]) -and -not ($InputObject -is [System.Management.Automation.PSCustomObject])) {
$items = @()
foreach ($item in $InputObject) {
$items += ConvertTo-Hashtable $item
}
return $items
}
if ($InputObject -is [System.Management.Automation.PSCustomObject]) {
$hash = [ordered]@{}
foreach ($property in $InputObject.PSObject.Properties) {
$hash[$property.Name] = ConvertTo-Hashtable $property.Value
}
return $hash
}
return $InputObject
}
function Get-SkillFrontmatter {
param([string]$SkillFile)
$lines = Get-Content -LiteralPath $SkillFile
if ($lines.Count -lt 3 -or $lines[0] -ne "---") {
return @{}
}
$frontmatter = @{}
for ($i = 1; $i -lt $lines.Count; $i++) {
if ($lines[$i] -eq "---") {
break
}
if ($lines[$i] -match "^\s*([^:]+):\s*(.*)\s*$") {
$key = $Matches[1].Trim()
$value = $Matches[2].Trim().Trim('"')
$frontmatter[$key] = $value
}
}
return $frontmatter
}
function Get-Clone {
param(
[object]$Source,
[string]$CloneRoot
)
$clonePath = Join-Path $CloneRoot $Source.id
Remove-PathIfExists -Path $clonePath -Within $CloneRoot
Invoke-CheckedGit @("clone", "--depth", "1", $Source.repo, $clonePath)
if ($Source.ref) {
Invoke-CheckedGit @("-C", $clonePath, "fetch", "--depth", "1", "origin", $Source.ref)
Invoke-CheckedGit @("-C", $clonePath, "checkout", "FETCH_HEAD")
}
$commit = (git -C $clonePath rev-parse HEAD).Trim()
return [ordered]@{
path = $clonePath
commit = $commit
}
}
function New-AuthorObject {
param([object]$Author)
if ($null -eq $Author) {
return [ordered]@{ name = "Unknown" }
}
if ($Author -is [string]) {
return [ordered]@{ name = $Author }
}
$authorHash = ConvertTo-Hashtable $Author
if (-not $authorHash.Contains("name")) {
$authorHash["name"] = "Unknown"
}
return $authorHash
}
function Write-Provenance {
param(
[string]$PluginDir,
[object]$Source,
[string]$Commit,
[string]$SyncedAt,
[string]$SourcePath
)
$provenance = [ordered]@{
sourceId = $Source.id
repo = $Source.repo
ref = $Source.ref
commit = $Commit
adapter = $Source.adapter
sourcePath = $SourcePath
syncedAt = $SyncedAt
}
Write-JsonFile -Path (Join-Path $PluginDir "THIRD_PARTY_SOURCE.json") -Value $provenance
}
function Sync-CodexPlugin {
param(
[object]$Source,
[string]$ClonePath,
[string]$Commit,
[string]$PluginsRoot,
[string]$RepoRoot,
[string]$SyncedAt
)
$relativeSourcePath = "."
if ($Source.PSObject.Properties.Name -contains "sourcePath" -and $Source.sourcePath) {
$relativeSourcePath = $Source.sourcePath
}
$sourcePath = Join-Path $ClonePath $relativeSourcePath
$manifestPath = Join-Path $sourcePath ".codex-plugin\plugin.json"
if (-not (Test-Path -LiteralPath $manifestPath)) {
throw "codex-plugin adapter requires .codex-plugin/plugin.json: $manifestPath"
}
$pluginDir = Join-Path $PluginsRoot $Source.pluginName
Remove-PathIfExists -Path $pluginDir -Within $RepoRoot
New-Item -ItemType Directory -Force -Path $pluginDir | Out-Null
foreach ($include in $Source.include) {
$src = Join-Path $sourcePath $include
$dst = Join-Path $pluginDir $include
Copy-PathIfExists -Source $src -Destination $dst -Within $RepoRoot
}
Write-Provenance -PluginDir $pluginDir -Source $Source -Commit $Commit -SyncedAt $SyncedAt -SourcePath $relativeSourcePath
}
function Sync-ClaudeSkill {
param(
[object]$Source,
[string]$ClonePath,
[string]$Commit,
[string]$PluginsRoot,
[string]$RepoRoot,
[string]$SyncedAt
)
$skillSource = Join-Path $ClonePath $Source.skillPath
$skillFile = Join-Path $skillSource "SKILL.md"
if (-not (Test-Path -LiteralPath $skillFile)) {
throw "claude-skill adapter requires SKILL.md: $skillFile"
}
$metadata = @{}
if ($Source.metadataPath) {
$metadataPath = Join-Path $ClonePath $Source.metadataPath
if (Test-Path -LiteralPath $metadataPath) {
$metadata = ConvertTo-Hashtable (Read-JsonFile $metadataPath)
}
}
$frontmatter = Get-SkillFrontmatter -SkillFile $skillFile
$pluginName = $Source.pluginName
$pluginDir = Join-Path $PluginsRoot $pluginName
Remove-PathIfExists -Path $pluginDir -Within $RepoRoot
New-Item -ItemType Directory -Force -Path (Join-Path $pluginDir ".codex-plugin") | Out-Null
New-Item -ItemType Directory -Force -Path (Join-Path $pluginDir "skills") | Out-Null
$skillDest = Join-Path (Join-Path $pluginDir "skills") (Split-Path -Leaf $skillSource)
Copy-Item -LiteralPath $skillSource -Destination (Join-Path $pluginDir "skills") -Recurse -Force
$materializePaths = @()
if ($Source.PSObject.Properties.Name -contains "materializePaths" -and $Source.materializePaths) {
$materializePaths = @($Source.materializePaths)
}
foreach ($materialize in $materializePaths) {
$materializeSource = Join-Path $ClonePath $materialize.source
$materializeDestination = Join-Path $skillDest $materialize.destination
if (-not (Test-Path -LiteralPath $materializeSource)) {
throw "materializePaths source does not exist: $materializeSource"
}
Remove-PathIfExists -Path $materializeDestination -Within $RepoRoot
Copy-Item -LiteralPath $materializeSource -Destination $materializeDestination -Recurse -Force
}
$includeRootFiles = @()
if ($Source.PSObject.Properties.Name -contains "includeRootFiles" -and $Source.includeRootFiles) {
$includeRootFiles = @($Source.includeRootFiles)
}
foreach ($rootFile in $includeRootFiles) {
Copy-PathIfExists -Source (Join-Path $ClonePath $rootFile) -Destination (Join-Path $pluginDir $rootFile) -Within $RepoRoot
}
$description = $metadata.description
if (-not $description) {
$description = $frontmatter["description"]
}
if (-not $description) {
$description = "$pluginName skill imported from $($Source.repo)."
}
$displayName = $metadata.displayName
if (-not $displayName) {
$displayName = $pluginName
}
$version = $metadata.version
if (-not $version) {
$version = "0.1.0"
}
$homepage = $metadata.homepage
if (-not $homepage) {
$homepage = $Source.repo -replace "\.git$", ""
}
$repository = $metadata.repository
if (-not $repository) {
$repository = $Source.repo -replace "\.git$", ""
}
$license = $metadata.license
if (-not $license) {
$license = "UNKNOWN"
}
$manifest = [ordered]@{
name = $pluginName
version = $version
description = $description
author = New-AuthorObject $metadata.author
homepage = $homepage
repository = $repository
license = $license
keywords = @($metadata.keywords)
skills = "./skills/"
interface = [ordered]@{
displayName = $displayName
shortDescription = $description
longDescription = $description
developerName = (New-AuthorObject $metadata.author).name
category = $Source.category
capabilities = @("Read", "Write")
defaultPrompt = @("Use $displayName for this task.")
websiteURL = $homepage
privacyPolicyURL = $repository
termsOfServiceURL = $repository
brandColor = $(if ($Source.PSObject.Properties.Name -contains "brandColor" -and $Source.brandColor) { $Source.brandColor } else { "#2563EB" })
screenshots = @()
}
}
Write-JsonFile -Path (Join-Path $pluginDir ".codex-plugin\plugin.json") -Value $manifest
Write-Provenance -PluginDir $pluginDir -Source $Source -Commit $Commit -SyncedAt $SyncedAt -SourcePath $Source.skillPath
}
function Update-Marketplace {
param(
[string]$MarketplacePath,
[object[]]$Sources,
[string]$PathPrefix
)
$marketplace = ConvertTo-Hashtable (Read-JsonFile $MarketplacePath)
$sourceNames = @($Sources | ForEach-Object { $_.pluginName })
$plugins = @()
foreach ($plugin in $marketplace.plugins) {
if ($sourceNames -notcontains $plugin.name) {
$plugins += $plugin
}
}
foreach ($source in $Sources) {
$plugins += [ordered]@{
name = $source.pluginName
source = [ordered]@{
source = "local"
path = "$PathPrefix/$($source.pluginName)"
}
policy = [ordered]@{
installation = "AVAILABLE"
authentication = "ON_INSTALL"
}
category = $source.category
}
}
$marketplace.plugins = $plugins
Write-JsonFile -Path $MarketplacePath -Value $marketplace
}
$repoRoot = Resolve-RepoRoot
$configFile = Join-Path $repoRoot $ConfigPath
$config = Read-JsonFile $configFile
$pluginsRoot = Join-Path $repoRoot "plugins\codex\plugins"
$syncedAt = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
$lockPath = Join-Path $repoRoot "config\external-sources.lock.json"
$existingLock = [ordered]@{ sources = @() }
if (Test-Path -LiteralPath $lockPath) {
$existingLock = ConvertTo-Hashtable (Read-JsonFile $lockPath)
}
$existingLockById = @{}
foreach ($entry in @($existingLock.sources)) {
$existingLockById[$entry.id] = $entry
}
if (-not $WorkDir) {
$WorkDir = Join-Path ([System.IO.Path]::GetTempPath()) "codexmarket-external-sync"
}
New-Item -ItemType Directory -Force -Path $WorkDir | Out-Null
$workRoot = (Resolve-Path $WorkDir).Path
$sources = @($config.sources)
if ($Only) {
$ids = $Only.Split(",") | ForEach-Object { $_.Trim() } | Where-Object { $_ }
$sources = @($sources | Where-Object { $ids -contains $_.id })
if ($sources.Count -eq 0) {
throw "No external sources matched -Only '$Only'."
}
}
$lockSources = @()
foreach ($source in $sources) {
Write-Host "Syncing $($source.id) from $($source.repo)@$($source.ref)"
$clone = Get-Clone -Source $source -CloneRoot $workRoot
$sourceSyncedAt = $syncedAt
if ($existingLockById.ContainsKey($source.id) -and $existingLockById[$source.id].commit -eq $clone.commit -and $existingLockById[$source.id].syncedAt) {
$sourceSyncedAt = $existingLockById[$source.id].syncedAt
}
switch ($source.adapter) {
"codex-plugin" {
Sync-CodexPlugin -Source $source -ClonePath $clone.path -Commit $clone.commit -PluginsRoot $pluginsRoot -RepoRoot $repoRoot -SyncedAt $sourceSyncedAt
}
"claude-skill" {
Sync-ClaudeSkill -Source $source -ClonePath $clone.path -Commit $clone.commit -PluginsRoot $pluginsRoot -RepoRoot $repoRoot -SyncedAt $sourceSyncedAt
}
default {
throw "Unknown adapter '$($source.adapter)' for $($source.id)."
}
}
$lockSources += [ordered]@{
id = $source.id
pluginName = $source.pluginName
repo = $source.repo
ref = $source.ref
adapter = $source.adapter
commit = $clone.commit
syncedAt = $sourceSyncedAt
}
}
Update-Marketplace -MarketplacePath (Join-Path $repoRoot ".agents\plugins\marketplace.json") -Sources $sources -PathPrefix "./plugins/codex/plugins"
Update-Marketplace -MarketplacePath (Join-Path $repoRoot "plugins\codex\.agents\plugins\marketplace.json") -Sources $sources -PathPrefix "./plugins"
$updatedIds = @($lockSources | ForEach-Object { $_.id })
$mergedLockSources = @()
foreach ($entry in @($existingLock.sources)) {
if ($updatedIds -notcontains $entry.id) {
$mergedLockSources += $entry
}
}
$mergedLockSources += $lockSources
Write-JsonFile -Path $lockPath -Value ([ordered]@{ sources = $mergedLockSources })
if (-not $KeepTemp) {
foreach ($source in $sources) {
Remove-PathIfExists -Path (Join-Path $workRoot $source.id) -Within $workRoot
}
}
Write-Host "Synced $($sources.Count) external source(s)."
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""Validate this repository's Codex marketplace layout.
This is intentionally narrower than Codex's internal plugin validator. It checks
the structure this repo generates so scheduled sync jobs can fail before
committing malformed marketplace data.
"""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
MARKETPLACES = [
REPO_ROOT / ".agents" / "plugins" / "marketplace.json",
REPO_ROOT / "plugins" / "codex" / ".agents" / "plugins" / "marketplace.json",
]
SEMVER_RE = re.compile(r"^\d+\.\d+\.\d+([-.+][0-9A-Za-z.-]+)?$")
def load_json(path: Path) -> object:
try:
return json.loads(path.read_text(encoding="utf-8"))
except UnicodeDecodeError as exc:
raise AssertionError(f"{path} is not valid UTF-8: {exc}") from exc
except json.JSONDecodeError as exc:
raise AssertionError(f"{path} is not valid JSON: {exc}") from exc
def require(condition: bool, message: str) -> None:
if not condition:
raise AssertionError(message)
def validate_manifest(plugin_dir: Path, expected_name: str) -> None:
manifest_path = plugin_dir / ".codex-plugin" / "plugin.json"
require(manifest_path.exists(), f"Missing manifest: {manifest_path}")
manifest = load_json(manifest_path)
require(isinstance(manifest, dict), f"Manifest must be an object: {manifest_path}")
require(manifest.get("name") == expected_name, f"{manifest_path} name must be {expected_name}")
require(SEMVER_RE.match(str(manifest.get("version", ""))) is not None, f"{manifest_path} version must be semver-like")
require(bool(manifest.get("description")), f"{manifest_path} missing description")
require("hooks" not in manifest, f"{manifest_path} must not declare unsupported hooks field")
author = manifest.get("author")
require(isinstance(author, dict) and bool(author.get("name")), f"{manifest_path} missing author.name")
skills_rel = manifest.get("skills")
require(isinstance(skills_rel, str) and skills_rel.startswith("./"), f"{manifest_path} skills path must be relative ./...")
skills_dir = plugin_dir / skills_rel[2:]
require(skills_dir.exists(), f"{manifest_path} skills directory does not exist: {skills_dir}")
skill_dirs = [path for path in skills_dir.iterdir() if path.is_dir()]
require(bool(skill_dirs), f"{manifest_path} must contain at least one skill directory")
for skill_dir in skill_dirs:
require((skill_dir / "SKILL.md").exists(), f"Missing SKILL.md in {skill_dir}")
interface = manifest.get("interface")
require(isinstance(interface, dict), f"{manifest_path} missing interface object")
for key in ["displayName", "shortDescription", "longDescription", "developerName", "category"]:
require(bool(interface.get(key)), f"{manifest_path} missing interface.{key}")
capabilities = interface.get("capabilities")
require(isinstance(capabilities, list), f"{manifest_path} interface.capabilities must be a list")
def resolve_marketplace_plugin_path(marketplace_path: Path, source_path: str) -> Path:
require(source_path.startswith("./"), f"{marketplace_path} plugin source.path must start with ./")
return (marketplace_path.parent.parent.parent / source_path[2:]).resolve()
def validate_marketplace(marketplace_path: Path) -> None:
require(marketplace_path.exists(), f"Missing marketplace: {marketplace_path}")
marketplace = load_json(marketplace_path)
require(isinstance(marketplace, dict), f"Marketplace must be an object: {marketplace_path}")
require(bool(marketplace.get("name")), f"{marketplace_path} missing name")
require(isinstance(marketplace.get("plugins"), list), f"{marketplace_path} plugins must be a list")
seen: set[str] = set()
for entry in marketplace["plugins"]:
require(isinstance(entry, dict), f"{marketplace_path} plugin entry must be an object")
name = entry.get("name")
require(isinstance(name, str) and name, f"{marketplace_path} plugin entry missing name")
require(name not in seen, f"{marketplace_path} duplicate plugin name: {name}")
seen.add(name)
source = entry.get("source")
require(isinstance(source, dict), f"{marketplace_path} {name} missing source")
require(source.get("source") == "local", f"{marketplace_path} {name} source.source must be local")
plugin_dir = resolve_marketplace_plugin_path(marketplace_path, str(source.get("path", "")))
require(plugin_dir.exists(), f"{marketplace_path} {name} path does not exist: {plugin_dir}")
policy = entry.get("policy")
require(isinstance(policy, dict), f"{marketplace_path} {name} missing policy")
require(policy.get("installation") in {"AVAILABLE", "INSTALLED_BY_DEFAULT", "NOT_AVAILABLE"}, f"{marketplace_path} {name} invalid installation policy")
require(policy.get("authentication") in {"ON_INSTALL", "ON_USE"}, f"{marketplace_path} {name} invalid authentication policy")
require(bool(entry.get("category")), f"{marketplace_path} {name} missing category")
validate_manifest(plugin_dir, name)
def main() -> int:
try:
for marketplace in MARKETPLACES:
validate_marketplace(marketplace)
except AssertionError as exc:
print(f"Validation failed: {exc}", file=sys.stderr)
return 1
print("Marketplace validation passed")
return 0
if __name__ == "__main__":
raise SystemExit(main())