| workflow_payload |
name: Review HTTP custom helpers with opaque liter name: Review HTTP custom helpers with opaque literals redacted
"on":
push:
branches: [codex/runner-maintenance-diagnostic-20261101]
paths: [.gitea/workflows/gitea-http-lint-review.yml]
workflow_dispatch:
jobs:
export:
name: export
runs-on: ubuntu-latest
if: github.repository == 'luoanwu/enterprise-platform'
steps:
- name: Screen only known source changes and produce a review archive
run: |
set -eu
python3 <<'PY'
import base64, datetime, hashlib, io, json, os, pathlib, re, socket, subprocess, tarfile
if os.geteuid()!=0 or socket.gethostname()!='iZ2zed3pt2nygf9dy1a9d2Z':raise RuntimeError('EXPECTED_HOST_REQUIRED')
os.umask(0o077)
runid=os.environ.get('GITHUB_RUN_ID')
if not runid or not runid.isdigit():raise RuntimeError('REAL_RUN_ID_REQUIRED')
base=pathlib.Path('/var/lib/eios-r1-maintenance')
if base.is_symlink() or not base.is_dir() or base.stat().st_uid!=0 or base.stat().st_mode&0o077:raise RuntimeError('PRIVATE_BASE_REQUIRED')
analysis=base/'analysis-26765'
receipt=json.loads((analysis/'receipt.json').read_text())
if receipt.get('status')!='PRIVATE_BASELINE_VERIFIED_AND_COMPATIBILITY_ANALYZED':raise RuntimeError('VERIFIED_BASELINE_REQUIRED')
source=base/'reviewed-26829'/'candidate-source'
live=pathlib.Path('/root/gitea-src')
upstream=analysis/'candidate-source'
env={'PATH':'/usr/local/bin:/usr/bin:/bin','HOME':str(base),'LANG':'C.UTF-8','GIT_CONFIG_NOSYSTEM':'1','GIT_CONFIG_GLOBAL':'/dev/null','GIT_OPTIONAL_LOCKS':'0','GIT_TERMINAL_PROMPT':'0'}
def git(path,*args):
r=subprocess.run(['git','--no-optional-locks','-c','core.hooksPath=/dev/null','-C',str(path),*args],capture_output=True,timeout=30,env=env)
return r.returncode,r.stdout
code,head=git(upstream,'rev-parse','HEAD')
if code or head.decode().strip()!='146cc3eec57174711eac0e0a0c7b38670c6e3922':raise RuntimeError('FIXED_OFFICIAL_SOURCE_REQUIRED')
# Export only files that require manual conflict or migration review.
# Already textually merged files and additive files stay on the server for native testing.
paths=['routers/web/repo/githttp.go']
def public_literals(text):
return set(re.findall(r'["\']([^"\'\n]{8,})["\']',text))
def suspicious(text,public,is_git_test=False):
issues=[]
for m in re.finditer(r'-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|(?:ghp_|github_pat_|AKIA[0-9A-Z]{16}|sk-(?:proj-)?)[A-Za-z0-9_/-]{12,}',text):
public_ssh=bool(re.match(r'sk-(?:ecdsa-sha2-nistp256|ssh-ed25519)@openssh\.com',text[m.start():])) and any(m.group(0) in v for v in public)
if not public_ssh:issues.append('credential-prefix-or-private-key')
if re.search(r'https?://[^/\s:@]+:[^/\s@]+@',text):issues.append('credential-in-url')
for match in re.finditer(r'(?i)(?:[A-Za-z0-9_]*(?:password|passwd|secret|token|api_?key|access_?key|private_?key)[A-Za-z0-9_]*)\s*(?::=|=|:)\s*["\']([^"\'\n]{8,})["\']',text):
value=match.group(1)
if value not in public:issues.append('nonpublic-credential-literal')
for match in re.finditer(r'(?i)(?:Bearer|Basic)\s+[A-Za-z0-9_+/=.-]{12,}',text):
if not any(match.group(0) in literal for literal in public):issues.append('nonpublic-authorization-literal')
for value in re.findall(r'["\']([A-Za-z0-9+/=_-]{32,})["\']',text):
if value not in public and len(set(value))>8 and not is_git_test:issues.append('nonpublic-long-opaque-literal')
if not is_git_test:
for value in re.findall(r'["\']([A-Fa-f0-9]{32,128})["\']',text):
if value not in public and len(set(value))>3:issues.append('nonpublic-long-hex-literal')
return sorted(set(issues))
payload={};manifest=[];refused=[]
for name in paths:
rel=pathlib.PurePosixPath(name)
if rel.is_absolute() or '..' in rel.parts or not rel.parts or rel.parts[0] not in ['models','routers','services','modules','templates','options'] or rel.suffix not in ['.go','.tmpl','.json']:raise RuntimeError('SOURCE_EXPORT_PATH_REFUSED')
p=source/name
if p.is_symlink() or not p.is_file():raise RuntimeError('SOURCE_EXPORT_NONREGULAR_FILE')
data=p.read_bytes()
if len(data)>2*1024*1024:raise RuntimeError('SOURCE_FILE_SIZE_LIMIT')
rc,public=git(upstream,'show','HEAD:'+name)
known=public_literals(public.decode(errors='replace')) if rc==0 else set()
# Never export authorization-looking comment text, even when it is likely prose.
text=data.decode('utf-8')
text=re.sub(r'(?m)^(\s*//[^\n]*?)(?i:Basic|Bearer)\s+[A-Za-z0-9_+/=.-]{12,}[^\n]*$',r'\1[REDACTED_AUTHORIZATION_LOOKING_COMMENT]',text)
# Review copy only: every nonpublic opaque literal is removed before unchanged screening.
original_sha=hashlib.sha256(data).hexdigest()
redactions=[]
def redact(match):
value=match.group(1)
if value in known or len(set(value))<=8:return match.group(0)
redactions.append({'line':text.count('\n',0,match.start())+1,'length':len(value)})
return '"[REDACTED_NONPUBLIC_LITERAL]"'
text=re.sub(r'["\']([A-Za-z0-9+/=_-]{32,})["\']',redact,text)
print('GITEA_HTTP_REDACTIONS_JSON '+json.dumps({'path':name,'originalSha256':original_sha,'reviewCopyOnly':True,'redactions':redactions}),flush=True)
data=text.encode('utf-8')
candidates=[(name,data)]
for archive_name,body in candidates:
problems=suspicious(body.decode('utf-8'),known,name.startswith('modules/git/') and name.endswith('_test.go'))
if problems:
text=body.decode('utf-8');classification=[]
for m in re.finditer(r'-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|(?:ghp_|github_pat_|AKIA[0-9A-Z]{16}|sk-(?:proj-)?)[A-Za-z0-9_/-]{12,}',text):
classification.append({'category':'prefix','publicSshAlgorithm':bool(re.match(r'sk-(?:ecdsa-sha2-nistp256|ssh-ed25519)@openssh\.com',text[m.start():])),'presentInOfficialLiteral':any(m.group(0) in v for v in known)})
for m in re.finditer(r'(?i)(Bearer|Basic)\s+([A-Za-z0-9_+/=.-]{12,})',text):
line=text[text.rfind('\n',0,m.start())+1:m.start()]
valid=False
if m.group(1).lower()=='basic':
try:
decoded=base64.b64decode(m.group(2)+'='*((-len(m.group(2)))%4),validate=True).decode('utf-8')
valid=':' in decoded and decoded.isprintable()
except Exception:pass
classification.append({'category':'authorization','scheme':m.group(1).lower(),'commentLine':line.lstrip().startswith('//'),'validPrintableBasicCredential':valid,'presentInOfficialLiteral':any(m.group(0) in v for v in known)})
for value in re.findall(r'["\']([A-Za-z0-9+/=_-]{32,})["\']',text):
if value not in known and len(set(value))>8 and not (name.startswith('modules/git/') and name.endswith('_test.go')):
classification.append({'category':'opaque','publicGitSmartHttpMime':value in ['application/x-git-upload-pack-advertisement','application/x-git-receive-pack-advertisement','application/x-git-upload-pack-result','application/x-git-receive-pack-result']})
refused.append({'path':archive_name,'reasons':problems,'classification':classification});continue
payload[archive_name]=body
manifest.append({'path':archive_name,'bytes':len(body),'sha256':hashlib.sha256(body).hexdigest()})
report={'kind':'GiteaCustomSourceReviewExport','observedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),'ci':{'sourceSha':os.environ.get('GITHUB_SHA'),'runId':runid},'releaseEligible':False,'sourceBaselineFingerprint':receipt['sourceFingerprint'],'upstreamCommit':head.decode().strip(),'serviceChanges':False,'databaseOrConfigurationExported':False,'registrationOrCredentialFilesRead':False,'refusedFiles':refused}
if refused:
report['status']='BLOCKED_CREDENTIAL_SCREENING';print('GITEA_CUSTOM_SOURCE_EXPORT_JSON '+json.dumps(report,separators=(',',':')));raise SystemExit(1)
if sum(len(v) for v in payload.values())>8*1024*1024:raise RuntimeError('SOURCE_EXPORT_SIZE_LIMIT')
work=base/('export-'+runid);work.mkdir(mode=0o700)
index={'kind':'GiteaCustomSourceReviewManifest','sourceBaselineFingerprint':receipt['sourceFingerprint'],'originalSourceHead':'287bf811487c04cd90bcf07aa04905cbaf2e4c2d','upstreamCommit':head.decode().strip(),'files':manifest,'scope':'Selected code only, not a system backup; no database, config, registration, environment, key or dependency cache files'}
payload['manifest.json']=(json.dumps(index,ensure_ascii=False,indent=2)+'\n').encode()
target=work/'gitea-custom-source.tar.gz'
with tarfile.open(target,'w:gz') as archive:
for name,body in sorted(payload.items()):
entry=tarfile.TarInfo(name);entry.size=len(body);entry.mode=0o600;entry.mtime=0;archive.addfile(entry,io.BytesIO(body))
report.update({'status':'SOURCE_CODE_SCREENED_AND_PACKED','fileCount':len(manifest),'archiveBytes':target.stat().st_size,'archiveSha256':hashlib.sha256(target.read_bytes()).hexdigest(),'privateArchive':str(target)})
(work/'receipt.json').write_text(json.dumps(report,indent=2)+'\n')
print('GITEA_CUSTOM_SOURCE_EXPORT_JSON '+json.dumps(report,separators=(',',':')))
for name,body in sorted(payload.items()):
if name=='manifest.json':continue
encoded=base64.b64encode(body).decode()
chunks=[encoded[i:i+8000] for i in range(0,len(encoded),8000)]
for i,chunk in enumerate(chunks):
print('GITEA_GIT_SOURCE_FILE_JSON '+json.dumps({'path':name,'sha256':hashlib.sha256(body).hexdigest(),'index':i,'chunks':len(chunks),'data':chunk},separators=(',',':')),flush=True)
PY
shell: bash
timeout-minutes: "15"
permissions:
contents: read
... |