Sync third-party skill plugins from upstream repos
Constraint: Maintainers need repeatable ingestion for external skill repositories with different layouts.\nRejected: Git submodules as the primary mechanism | Codex marketplace installs do not reliably recurse external submodules, and adapters still need layout conversion.\nConfidence: medium\nScope-risk: moderate\nDirective: Add new upstreams through config/external-sources.json and adapter logic rather than hand-copying plugin folders.\nTested: scripts/sync_external_plugins.ps1 -Proxy http://127.0.0.1:18085; scripts/sync_external_plugins.ps1 -Only ui-ux-pro-max -Proxy http://127.0.0.1:18085; python scripts/validate_marketplace.py; validate_plugin.py for superpowers and ui-ux-pro-max; codex plugin add superpowers@eapil-skill-market; codex plugin add ui-ux-pro-max@eapil-skill-market; verified ui-ux-pro-max data/scripts directories in cache.\nNot-tested: Scheduled .gitea workflow execution on git.playones runner.
This commit is contained in:
AreChen
@@ -0,0 +1,429 @@
|
||||
param(
|
||||
[string]$ConfigPath = "config/external-sources.json",
|
||||
[string]$WorkDir = "",
|
||||
[string]$Only = "",
|
||||
[string]$Proxy = "",
|
||||
[switch]$KeepTemp
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
if ($Proxy) {
|
||||
$env:HTTP_PROXY = $Proxy
|
||||
$env:HTTPS_PROXY = $Proxy
|
||||
$env:ALL_PROXY = $Proxy
|
||||
}
|
||||
|
||||
function Invoke-CheckedGit {
|
||||
param([string[]]$GitArgs)
|
||||
& git @GitArgs
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "git $($GitArgs -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Resolve-RepoRoot {
|
||||
$root = git rev-parse --show-toplevel
|
||||
if (-not $root) {
|
||||
throw "Run this script from inside a git repository."
|
||||
}
|
||||
return (Resolve-Path $root.Trim()).Path
|
||||
}
|
||||
|
||||
function Assert-ChildPath {
|
||||
param(
|
||||
[string]$Parent,
|
||||
[string]$Child
|
||||
)
|
||||
$parentPath = (Resolve-Path $Parent).Path
|
||||
$childPath = $Child
|
||||
if (Test-Path -LiteralPath $Child) {
|
||||
$childPath = (Resolve-Path $Child).Path
|
||||
} else {
|
||||
$childPath = [System.IO.Path]::GetFullPath($Child)
|
||||
}
|
||||
if (-not $childPath.StartsWith($parentPath, [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "Refusing to write outside $parentPath`: $childPath"
|
||||
}
|
||||
}
|
||||
|
||||
function Remove-PathIfExists {
|
||||
param([string]$Path, [string]$Within)
|
||||
if (Test-Path -LiteralPath $Path) {
|
||||
Assert-ChildPath -Parent $Within -Child $Path
|
||||
Remove-Item -LiteralPath $Path -Recurse -Force
|
||||
}
|
||||
}
|
||||
|
||||
function Copy-PathIfExists {
|
||||
param(
|
||||
[string]$Source,
|
||||
[string]$Destination,
|
||||
[string]$Within
|
||||
)
|
||||
if (Test-Path -LiteralPath $Source) {
|
||||
Assert-ChildPath -Parent $Within -Child $Destination
|
||||
Copy-Item -LiteralPath $Source -Destination $Destination -Recurse -Force
|
||||
}
|
||||
}
|
||||
|
||||
function Read-JsonFile {
|
||||
param([string]$Path)
|
||||
return Get-Content -Raw -LiteralPath $Path | ConvertFrom-Json
|
||||
}
|
||||
|
||||
function Write-JsonFile {
|
||||
param([string]$Path, [object]$Value)
|
||||
$json = $Value | ConvertTo-Json -Depth 20
|
||||
$utf8NoBom = New-Object System.Text.UTF8Encoding($false)
|
||||
[System.IO.File]::WriteAllText($Path, "$json`n", $utf8NoBom)
|
||||
}
|
||||
|
||||
function ConvertTo-Hashtable {
|
||||
param([object]$InputObject)
|
||||
if ($null -eq $InputObject) {
|
||||
return $null
|
||||
}
|
||||
if ($InputObject -is [System.Collections.IEnumerable] -and -not ($InputObject -is [string]) -and -not ($InputObject -is [System.Management.Automation.PSCustomObject])) {
|
||||
$items = @()
|
||||
foreach ($item in $InputObject) {
|
||||
$items += ConvertTo-Hashtable $item
|
||||
}
|
||||
return $items
|
||||
}
|
||||
if ($InputObject -is [System.Management.Automation.PSCustomObject]) {
|
||||
$hash = [ordered]@{}
|
||||
foreach ($property in $InputObject.PSObject.Properties) {
|
||||
$hash[$property.Name] = ConvertTo-Hashtable $property.Value
|
||||
}
|
||||
return $hash
|
||||
}
|
||||
return $InputObject
|
||||
}
|
||||
|
||||
function Get-SkillFrontmatter {
|
||||
param([string]$SkillFile)
|
||||
$lines = Get-Content -LiteralPath $SkillFile
|
||||
if ($lines.Count -lt 3 -or $lines[0] -ne "---") {
|
||||
return @{}
|
||||
}
|
||||
$frontmatter = @{}
|
||||
for ($i = 1; $i -lt $lines.Count; $i++) {
|
||||
if ($lines[$i] -eq "---") {
|
||||
break
|
||||
}
|
||||
if ($lines[$i] -match "^\s*([^:]+):\s*(.*)\s*$") {
|
||||
$key = $Matches[1].Trim()
|
||||
$value = $Matches[2].Trim().Trim('"')
|
||||
$frontmatter[$key] = $value
|
||||
}
|
||||
}
|
||||
return $frontmatter
|
||||
}
|
||||
|
||||
function Get-Clone {
|
||||
param(
|
||||
[object]$Source,
|
||||
[string]$CloneRoot
|
||||
)
|
||||
$clonePath = Join-Path $CloneRoot $Source.id
|
||||
Remove-PathIfExists -Path $clonePath -Within $CloneRoot
|
||||
Invoke-CheckedGit @("clone", "--depth", "1", $Source.repo, $clonePath)
|
||||
if ($Source.ref) {
|
||||
Invoke-CheckedGit @("-C", $clonePath, "fetch", "--depth", "1", "origin", $Source.ref)
|
||||
Invoke-CheckedGit @("-C", $clonePath, "checkout", "FETCH_HEAD")
|
||||
}
|
||||
$commit = (git -C $clonePath rev-parse HEAD).Trim()
|
||||
return [ordered]@{
|
||||
path = $clonePath
|
||||
commit = $commit
|
||||
}
|
||||
}
|
||||
|
||||
function New-AuthorObject {
|
||||
param([object]$Author)
|
||||
if ($null -eq $Author) {
|
||||
return [ordered]@{ name = "Unknown" }
|
||||
}
|
||||
if ($Author -is [string]) {
|
||||
return [ordered]@{ name = $Author }
|
||||
}
|
||||
$authorHash = ConvertTo-Hashtable $Author
|
||||
if (-not $authorHash.Contains("name")) {
|
||||
$authorHash["name"] = "Unknown"
|
||||
}
|
||||
return $authorHash
|
||||
}
|
||||
|
||||
function Write-Provenance {
|
||||
param(
|
||||
[string]$PluginDir,
|
||||
[object]$Source,
|
||||
[string]$Commit,
|
||||
[string]$SyncedAt,
|
||||
[string]$SourcePath
|
||||
)
|
||||
$provenance = [ordered]@{
|
||||
sourceId = $Source.id
|
||||
repo = $Source.repo
|
||||
ref = $Source.ref
|
||||
commit = $Commit
|
||||
adapter = $Source.adapter
|
||||
sourcePath = $SourcePath
|
||||
syncedAt = $SyncedAt
|
||||
}
|
||||
Write-JsonFile -Path (Join-Path $PluginDir "THIRD_PARTY_SOURCE.json") -Value $provenance
|
||||
}
|
||||
|
||||
function Sync-CodexPlugin {
|
||||
param(
|
||||
[object]$Source,
|
||||
[string]$ClonePath,
|
||||
[string]$Commit,
|
||||
[string]$PluginsRoot,
|
||||
[string]$RepoRoot,
|
||||
[string]$SyncedAt
|
||||
)
|
||||
$relativeSourcePath = "."
|
||||
if ($Source.PSObject.Properties.Name -contains "sourcePath" -and $Source.sourcePath) {
|
||||
$relativeSourcePath = $Source.sourcePath
|
||||
}
|
||||
$sourcePath = Join-Path $ClonePath $relativeSourcePath
|
||||
$manifestPath = Join-Path $sourcePath ".codex-plugin\plugin.json"
|
||||
if (-not (Test-Path -LiteralPath $manifestPath)) {
|
||||
throw "codex-plugin adapter requires .codex-plugin/plugin.json: $manifestPath"
|
||||
}
|
||||
$pluginDir = Join-Path $PluginsRoot $Source.pluginName
|
||||
Remove-PathIfExists -Path $pluginDir -Within $RepoRoot
|
||||
New-Item -ItemType Directory -Force -Path $pluginDir | Out-Null
|
||||
foreach ($include in $Source.include) {
|
||||
$src = Join-Path $sourcePath $include
|
||||
$dst = Join-Path $pluginDir $include
|
||||
Copy-PathIfExists -Source $src -Destination $dst -Within $RepoRoot
|
||||
}
|
||||
Write-Provenance -PluginDir $pluginDir -Source $Source -Commit $Commit -SyncedAt $SyncedAt -SourcePath $relativeSourcePath
|
||||
}
|
||||
|
||||
function Sync-ClaudeSkill {
|
||||
param(
|
||||
[object]$Source,
|
||||
[string]$ClonePath,
|
||||
[string]$Commit,
|
||||
[string]$PluginsRoot,
|
||||
[string]$RepoRoot,
|
||||
[string]$SyncedAt
|
||||
)
|
||||
$skillSource = Join-Path $ClonePath $Source.skillPath
|
||||
$skillFile = Join-Path $skillSource "SKILL.md"
|
||||
if (-not (Test-Path -LiteralPath $skillFile)) {
|
||||
throw "claude-skill adapter requires SKILL.md: $skillFile"
|
||||
}
|
||||
|
||||
$metadata = @{}
|
||||
if ($Source.metadataPath) {
|
||||
$metadataPath = Join-Path $ClonePath $Source.metadataPath
|
||||
if (Test-Path -LiteralPath $metadataPath) {
|
||||
$metadata = ConvertTo-Hashtable (Read-JsonFile $metadataPath)
|
||||
}
|
||||
}
|
||||
$frontmatter = Get-SkillFrontmatter -SkillFile $skillFile
|
||||
|
||||
$pluginName = $Source.pluginName
|
||||
$pluginDir = Join-Path $PluginsRoot $pluginName
|
||||
Remove-PathIfExists -Path $pluginDir -Within $RepoRoot
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $pluginDir ".codex-plugin") | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $pluginDir "skills") | Out-Null
|
||||
$skillDest = Join-Path (Join-Path $pluginDir "skills") (Split-Path -Leaf $skillSource)
|
||||
Copy-Item -LiteralPath $skillSource -Destination (Join-Path $pluginDir "skills") -Recurse -Force
|
||||
|
||||
$materializePaths = @()
|
||||
if ($Source.PSObject.Properties.Name -contains "materializePaths" -and $Source.materializePaths) {
|
||||
$materializePaths = @($Source.materializePaths)
|
||||
}
|
||||
foreach ($materialize in $materializePaths) {
|
||||
$materializeSource = Join-Path $ClonePath $materialize.source
|
||||
$materializeDestination = Join-Path $skillDest $materialize.destination
|
||||
if (-not (Test-Path -LiteralPath $materializeSource)) {
|
||||
throw "materializePaths source does not exist: $materializeSource"
|
||||
}
|
||||
Remove-PathIfExists -Path $materializeDestination -Within $RepoRoot
|
||||
Copy-Item -LiteralPath $materializeSource -Destination $materializeDestination -Recurse -Force
|
||||
}
|
||||
|
||||
$includeRootFiles = @()
|
||||
if ($Source.PSObject.Properties.Name -contains "includeRootFiles" -and $Source.includeRootFiles) {
|
||||
$includeRootFiles = @($Source.includeRootFiles)
|
||||
}
|
||||
foreach ($rootFile in $includeRootFiles) {
|
||||
Copy-PathIfExists -Source (Join-Path $ClonePath $rootFile) -Destination (Join-Path $pluginDir $rootFile) -Within $RepoRoot
|
||||
}
|
||||
|
||||
$description = $metadata.description
|
||||
if (-not $description) {
|
||||
$description = $frontmatter["description"]
|
||||
}
|
||||
if (-not $description) {
|
||||
$description = "$pluginName skill imported from $($Source.repo)."
|
||||
}
|
||||
$displayName = $metadata.displayName
|
||||
if (-not $displayName) {
|
||||
$displayName = $pluginName
|
||||
}
|
||||
$version = $metadata.version
|
||||
if (-not $version) {
|
||||
$version = "0.1.0"
|
||||
}
|
||||
$homepage = $metadata.homepage
|
||||
if (-not $homepage) {
|
||||
$homepage = $Source.repo -replace "\.git$", ""
|
||||
}
|
||||
$repository = $metadata.repository
|
||||
if (-not $repository) {
|
||||
$repository = $Source.repo -replace "\.git$", ""
|
||||
}
|
||||
$license = $metadata.license
|
||||
if (-not $license) {
|
||||
$license = "UNKNOWN"
|
||||
}
|
||||
|
||||
$manifest = [ordered]@{
|
||||
name = $pluginName
|
||||
version = $version
|
||||
description = $description
|
||||
author = New-AuthorObject $metadata.author
|
||||
homepage = $homepage
|
||||
repository = $repository
|
||||
license = $license
|
||||
keywords = @($metadata.keywords)
|
||||
skills = "./skills/"
|
||||
interface = [ordered]@{
|
||||
displayName = $displayName
|
||||
shortDescription = $description
|
||||
longDescription = $description
|
||||
developerName = (New-AuthorObject $metadata.author).name
|
||||
category = $Source.category
|
||||
capabilities = @("Read", "Write")
|
||||
defaultPrompt = @("Use $displayName for this task.")
|
||||
websiteURL = $homepage
|
||||
privacyPolicyURL = $repository
|
||||
termsOfServiceURL = $repository
|
||||
brandColor = $(if ($Source.PSObject.Properties.Name -contains "brandColor" -and $Source.brandColor) { $Source.brandColor } else { "#2563EB" })
|
||||
screenshots = @()
|
||||
}
|
||||
}
|
||||
Write-JsonFile -Path (Join-Path $pluginDir ".codex-plugin\plugin.json") -Value $manifest
|
||||
Write-Provenance -PluginDir $pluginDir -Source $Source -Commit $Commit -SyncedAt $SyncedAt -SourcePath $Source.skillPath
|
||||
}
|
||||
|
||||
function Update-Marketplace {
|
||||
param(
|
||||
[string]$MarketplacePath,
|
||||
[object[]]$Sources,
|
||||
[string]$PathPrefix
|
||||
)
|
||||
$marketplace = ConvertTo-Hashtable (Read-JsonFile $MarketplacePath)
|
||||
$sourceNames = @($Sources | ForEach-Object { $_.pluginName })
|
||||
$plugins = @()
|
||||
foreach ($plugin in $marketplace.plugins) {
|
||||
if ($sourceNames -notcontains $plugin.name) {
|
||||
$plugins += $plugin
|
||||
}
|
||||
}
|
||||
foreach ($source in $Sources) {
|
||||
$plugins += [ordered]@{
|
||||
name = $source.pluginName
|
||||
source = [ordered]@{
|
||||
source = "local"
|
||||
path = "$PathPrefix/$($source.pluginName)"
|
||||
}
|
||||
policy = [ordered]@{
|
||||
installation = "AVAILABLE"
|
||||
authentication = "ON_INSTALL"
|
||||
}
|
||||
category = $source.category
|
||||
}
|
||||
}
|
||||
$marketplace.plugins = $plugins
|
||||
Write-JsonFile -Path $MarketplacePath -Value $marketplace
|
||||
}
|
||||
|
||||
$repoRoot = Resolve-RepoRoot
|
||||
$configFile = Join-Path $repoRoot $ConfigPath
|
||||
$config = Read-JsonFile $configFile
|
||||
$pluginsRoot = Join-Path $repoRoot "plugins\codex\plugins"
|
||||
$syncedAt = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
|
||||
$lockPath = Join-Path $repoRoot "config\external-sources.lock.json"
|
||||
$existingLock = [ordered]@{ sources = @() }
|
||||
if (Test-Path -LiteralPath $lockPath) {
|
||||
$existingLock = ConvertTo-Hashtable (Read-JsonFile $lockPath)
|
||||
}
|
||||
$existingLockById = @{}
|
||||
foreach ($entry in @($existingLock.sources)) {
|
||||
$existingLockById[$entry.id] = $entry
|
||||
}
|
||||
|
||||
if (-not $WorkDir) {
|
||||
$WorkDir = Join-Path ([System.IO.Path]::GetTempPath()) "codexmarket-external-sync"
|
||||
}
|
||||
New-Item -ItemType Directory -Force -Path $WorkDir | Out-Null
|
||||
$workRoot = (Resolve-Path $WorkDir).Path
|
||||
|
||||
$sources = @($config.sources)
|
||||
if ($Only) {
|
||||
$ids = $Only.Split(",") | ForEach-Object { $_.Trim() } | Where-Object { $_ }
|
||||
$sources = @($sources | Where-Object { $ids -contains $_.id })
|
||||
if ($sources.Count -eq 0) {
|
||||
throw "No external sources matched -Only '$Only'."
|
||||
}
|
||||
}
|
||||
|
||||
$lockSources = @()
|
||||
foreach ($source in $sources) {
|
||||
Write-Host "Syncing $($source.id) from $($source.repo)@$($source.ref)"
|
||||
$clone = Get-Clone -Source $source -CloneRoot $workRoot
|
||||
$sourceSyncedAt = $syncedAt
|
||||
if ($existingLockById.ContainsKey($source.id) -and $existingLockById[$source.id].commit -eq $clone.commit -and $existingLockById[$source.id].syncedAt) {
|
||||
$sourceSyncedAt = $existingLockById[$source.id].syncedAt
|
||||
}
|
||||
switch ($source.adapter) {
|
||||
"codex-plugin" {
|
||||
Sync-CodexPlugin -Source $source -ClonePath $clone.path -Commit $clone.commit -PluginsRoot $pluginsRoot -RepoRoot $repoRoot -SyncedAt $sourceSyncedAt
|
||||
}
|
||||
"claude-skill" {
|
||||
Sync-ClaudeSkill -Source $source -ClonePath $clone.path -Commit $clone.commit -PluginsRoot $pluginsRoot -RepoRoot $repoRoot -SyncedAt $sourceSyncedAt
|
||||
}
|
||||
default {
|
||||
throw "Unknown adapter '$($source.adapter)' for $($source.id)."
|
||||
}
|
||||
}
|
||||
$lockSources += [ordered]@{
|
||||
id = $source.id
|
||||
pluginName = $source.pluginName
|
||||
repo = $source.repo
|
||||
ref = $source.ref
|
||||
adapter = $source.adapter
|
||||
commit = $clone.commit
|
||||
syncedAt = $sourceSyncedAt
|
||||
}
|
||||
}
|
||||
|
||||
Update-Marketplace -MarketplacePath (Join-Path $repoRoot ".agents\plugins\marketplace.json") -Sources $sources -PathPrefix "./plugins/codex/plugins"
|
||||
Update-Marketplace -MarketplacePath (Join-Path $repoRoot "plugins\codex\.agents\plugins\marketplace.json") -Sources $sources -PathPrefix "./plugins"
|
||||
|
||||
$updatedIds = @($lockSources | ForEach-Object { $_.id })
|
||||
$mergedLockSources = @()
|
||||
foreach ($entry in @($existingLock.sources)) {
|
||||
if ($updatedIds -notcontains $entry.id) {
|
||||
$mergedLockSources += $entry
|
||||
}
|
||||
}
|
||||
$mergedLockSources += $lockSources
|
||||
Write-JsonFile -Path $lockPath -Value ([ordered]@{ sources = $mergedLockSources })
|
||||
|
||||
if (-not $KeepTemp) {
|
||||
foreach ($source in $sources) {
|
||||
Remove-PathIfExists -Path (Join-Path $workRoot $source.id) -Within $workRoot
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Synced $($sources.Count) external source(s)."
|
||||
Reference in New Issue
Block a user