main
1package flux
2
3import (
4 "context"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net/http"
9 "net/url"
10 "os"
11 "strings"
12 "time"
13)
14
15// GitHubSource fetches PRs and issues authored by a user,
16// and releases from watched orgs.
17type GitHubSource struct {
18 User string
19 Token string // optional, for higher rate limits
20 ReleaseOrgs []string // orgs to watch for releases (e.g. ["tektoncd", "vdemeester"])
21 Client *http.Client
22}
23
24// githubSearchResult represents the GitHub search API response.
25type githubSearchResult struct {
26 TotalCount int `json:"total_count"`
27 Items []githubItem `json:"items"`
28}
29
30type githubItem struct {
31 ID int `json:"id"`
32 Number int `json:"number"`
33 Title string `json:"title"`
34 HTMLURL string `json:"html_url"`
35 State string `json:"state"`
36 CreatedAt time.Time `json:"created_at"`
37 ClosedAt *time.Time `json:"closed_at"`
38 MergedAt *time.Time `json:"merged_at,omitempty"`
39 Body string `json:"body"`
40 Labels []struct {
41 Name string `json:"name"`
42 } `json:"labels"`
43 PullRequest *struct {
44 MergedAt *time.Time `json:"merged_at"`
45 } `json:"pull_request,omitempty"`
46 Repository *struct {
47 FullName string `json:"full_name"`
48 } `json:"repository,omitempty"`
49 RepositoryURL string `json:"repository_url"`
50}
51
52func (g *GitHubSource) Name() string { return "github" }
53
54func (g *GitHubSource) client() *http.Client {
55 if g.Client != nil {
56 return g.Client
57 }
58 return &http.Client{Timeout: 30 * time.Second}
59}
60
61// Fetch retrieves merged PRs and created issues authored by the configured user.
62func (g *GitHubSource) Fetch(ctx context.Context, since time.Time) ([]Entry, error) {
63 // Default lookback: 1 year if no since given (avoids GitHub timeouts on huge queries)
64 if since.IsZero() {
65 since = time.Now().AddDate(-1, 0, 0)
66 }
67
68 var entries []Entry
69 var errs []error
70
71 // Fetch merged PRs only — the meaningful event is the merge
72 prs, err := g.searchIssues(ctx, "type:pr is:merged", since)
73 if err != nil {
74 errs = append(errs, fmt.Errorf("fetching PRs: %w", err))
75 } else {
76 entries = append(entries, prs...)
77 }
78
79 // Fetch issues — track creation only (not state changes)
80 issues, err := g.searchIssues(ctx, "type:issue", since)
81 if err != nil {
82 errs = append(errs, fmt.Errorf("fetching issues: %w", err))
83 } else {
84 entries = append(entries, issues...)
85 }
86
87 // Fetch releases from watched orgs
88 for _, org := range g.ReleaseOrgs {
89 releases, err := g.fetchOrgReleases(ctx, org, since)
90 if err != nil {
91 errs = append(errs, fmt.Errorf("fetching releases for %s: %w", org, err))
92 } else {
93 entries = append(entries, releases...)
94 }
95 }
96
97 // Return what we got, even if some sources failed
98 if len(entries) == 0 && len(errs) > 0 {
99 return nil, errs[0]
100 }
101 for _, e := range errs {
102 fmt.Fprintf(os.Stderr, "warning: %v\n", e)
103 }
104
105 return entries, nil
106}
107
108func (g *GitHubSource) searchIssues(ctx context.Context, typeFilter string, since time.Time) ([]Entry, error) {
109 var allEntries []Entry
110 page := 1
111
112 for {
113 q := fmt.Sprintf("author:%s %s", g.User, typeFilter)
114 if !since.IsZero() {
115 q += fmt.Sprintf(" created:>%s", since.Format("2006-01-02"))
116 }
117
118 u := fmt.Sprintf("https://api.github.com/search/issues?q=%s&sort=created&order=desc&per_page=100&page=%d",
119 url.QueryEscape(q), page)
120
121 result, err := g.doRequest(ctx, u)
122 if err != nil {
123 return nil, err
124 }
125
126 for _, item := range result.Items {
127 entries := g.itemToEntry(item)
128 allEntries = append(allEntries, entries)
129 }
130
131 // Stop if we got fewer than a full page
132 if len(result.Items) < 100 {
133 break
134 }
135 page++
136
137 // Safety: don't paginate forever
138 if page > 10 {
139 break
140 }
141 }
142
143 return allEntries, nil
144}
145
146func (g *GitHubSource) doRequest(ctx context.Context, u string) (*githubSearchResult, error) {
147 req, err := http.NewRequestWithContext(ctx, "GET", u, nil)
148 if err != nil {
149 return nil, fmt.Errorf("creating request: %w", err)
150 }
151
152 req.Header.Set("Accept", "application/vnd.github+json")
153 if g.Token != "" {
154 req.Header.Set("Authorization", "Bearer "+g.Token)
155 }
156
157 resp, err := g.client().Do(req)
158 if err != nil {
159 return nil, fmt.Errorf("executing request: %w", err)
160 }
161 defer resp.Body.Close()
162
163 if resp.StatusCode != http.StatusOK {
164 body, _ := io.ReadAll(resp.Body)
165 return nil, fmt.Errorf("GitHub API error %d: %s", resp.StatusCode, string(body))
166 }
167
168 var result githubSearchResult
169 if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
170 return nil, fmt.Errorf("decoding response: %w", err)
171 }
172
173 return &result, nil
174}
175
176func (g *GitHubSource) itemToEntry(item githubItem) Entry {
177 isPR := item.PullRequest != nil
178
179 var kind EntryKind
180 var state string
181 var date time.Time
182
183 if isPR {
184 // PRs are always merged (we only search is:merged)
185 kind = KindGitHubPR
186 state = "merged"
187 if item.PullRequest != nil && item.PullRequest.MergedAt != nil {
188 date = *item.PullRequest.MergedAt
189 } else {
190 date = item.CreatedAt
191 }
192 } else {
193 // Issues track creation only
194 kind = KindGitHubIssue
195 state = "opened"
196 date = item.CreatedAt
197 }
198
199 // Extract repo name from repository_url
200 // Format: https://api.github.com/repos/owner/repo
201 repo := ""
202 if item.Repository != nil {
203 repo = item.Repository.FullName
204 } else if item.RepositoryURL != "" {
205 parts := strings.Split(item.RepositoryURL, "/")
206 if len(parts) >= 2 {
207 repo = parts[len(parts)-2] + "/" + parts[len(parts)-1]
208 }
209 }
210
211 id := fmt.Sprintf("%s-%s#%d", kind, repo, item.Number)
212
213 // Extract tags from labels
214 var tags []string
215 if repo != "" {
216 // Add org as tag, but skip personal usernames (only keep real orgs)
217 if parts := strings.SplitN(repo, "/", 2); len(parts) == 2 {
218 org := parts[0]
219 if !isGitHubUsername(org) {
220 tags = append(tags, strings.ToLower(org))
221 }
222 }
223 }
224 for _, label := range item.Labels {
225 name := label.Name
226
227 // --- Skip noise ---
228 // Nixpkgs bot labels (numbered prefixes, CI metadata)
229 if len(name) > 1 && name[0] >= '0' && name[0] <= '9' {
230 continue
231 }
232 // Workflow / CI labels
233 switch name {
234 case "approved", "lgtm", "authorized-changes",
235 "installation-validated", "package-validated",
236 "release-note-none", "automated",
237 "Tests", "tests":
238 continue
239 }
240 // Skip reviewer/maintainer usernames (from nixpkgs)
241 if isGitHubUsername(name) {
242 continue
243 }
244 // Size, priority, area prefixes — too project-specific
245 if strings.HasPrefix(name, "size/") ||
246 strings.HasPrefix(name, "priority/") ||
247 strings.HasPrefix(name, "area/") ||
248 strings.Contains(name, "rebuild-") {
249 continue
250 }
251
252 // --- Transform kind/* → strip prefix, skip "misc" ---
253 if strings.HasPrefix(name, "kind/") {
254 val := name[5:]
255 if val == "misc" {
256 continue
257 }
258 tags = append(tags, strings.ToLower(val))
259 continue
260 }
261
262 // Sanitize remaining labels
263 tag := strings.ReplaceAll(name, "/", "-")
264 tag = strings.ReplaceAll(tag, " ", "-")
265 tag = strings.ReplaceAll(tag, ":", "")
266 tag = strings.ToLower(tag)
267 tags = append(tags, tag)
268 }
269
270 metadata := map[string]string{
271 "state": state,
272 }
273 if repo != "" {
274 metadata["repo"] = repo
275 metadata["number"] = fmt.Sprintf("%d", item.Number)
276 }
277
278 return Entry{
279 ID: id,
280 Kind: kind,
281 Title: item.Title,
282 URL: item.HTMLURL,
283 Tags: tags,
284 Date: date,
285 Source: "github",
286 Metadata: metadata,
287 }
288}
289
290// --- Release fetching ---
291
292type githubRepo struct {
293 FullName string `json:"full_name"`
294}
295
296type githubRelease struct {
297 ID int `json:"id"`
298 TagName string `json:"tag_name"`
299 Name string `json:"name"`
300 HTMLURL string `json:"html_url"`
301 PublishedAt *time.Time `json:"published_at"`
302 CreatedAt time.Time `json:"created_at"`
303 Draft bool `json:"draft"`
304 Prerelease bool `json:"prerelease"`
305}
306
307// fetchOrgReleases lists repos for an org, then fetches recent releases from each.
308func (g *GitHubSource) fetchOrgReleases(ctx context.Context, org string, since time.Time) ([]Entry, error) {
309 repos, err := g.listOrgRepos(ctx, org)
310 if err != nil {
311 return nil, err
312 }
313
314 var entries []Entry
315 for _, repo := range repos {
316 releases, err := g.fetchRepoReleases(ctx, repo, since)
317 if err != nil {
318 // Log and continue — don't fail on one repo
319 fmt.Fprintf(os.Stderr, "warning: releases for %s: %v\n", repo, err)
320 continue
321 }
322 entries = append(entries, releases...)
323 }
324 return entries, nil
325}
326
327// listOrgRepos lists all repos for an org or user.
328func (g *GitHubSource) listOrgRepos(ctx context.Context, org string) ([]string, error) {
329 // Try as org first, fall back to user
330 repos, err := g.listReposAt(ctx, fmt.Sprintf("https://api.github.com/orgs/%s/repos", org))
331 if err != nil {
332 // Might be a user, not an org
333 repos, err = g.listReposAt(ctx, fmt.Sprintf("https://api.github.com/users/%s/repos", org))
334 }
335 return repos, err
336}
337
338func (g *GitHubSource) listReposAt(ctx context.Context, baseURL string) ([]string, error) {
339 var allRepos []string
340 page := 1
341 for {
342 u := fmt.Sprintf("%s?per_page=100&type=public&page=%d", baseURL, page)
343 req, err := http.NewRequestWithContext(ctx, "GET", u, nil)
344 if err != nil {
345 return nil, err
346 }
347 req.Header.Set("Accept", "application/vnd.github+json")
348 if g.Token != "" {
349 req.Header.Set("Authorization", "Bearer "+g.Token)
350 }
351 resp, err := g.client().Do(req)
352 if err != nil {
353 return nil, err
354 }
355 defer resp.Body.Close()
356 if resp.StatusCode != 200 {
357 return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
358 }
359 var repos []githubRepo
360 if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil {
361 return nil, err
362 }
363 for _, r := range repos {
364 allRepos = append(allRepos, r.FullName)
365 }
366 if len(repos) < 100 {
367 break
368 }
369 page++
370 }
371 return allRepos, nil
372}
373
374func (g *GitHubSource) fetchRepoReleases(ctx context.Context, repo string, since time.Time) ([]Entry, error) {
375 u := fmt.Sprintf("https://api.github.com/repos/%s/releases?per_page=20", repo)
376 req, err := http.NewRequestWithContext(ctx, "GET", u, nil)
377 if err != nil {
378 return nil, err
379 }
380 req.Header.Set("Accept", "application/vnd.github+json")
381 if g.Token != "" {
382 req.Header.Set("Authorization", "Bearer "+g.Token)
383 }
384
385 resp, err := g.client().Do(req)
386 if err != nil {
387 return nil, err
388 }
389 defer resp.Body.Close()
390
391 if resp.StatusCode == 404 {
392 // No releases or private repo — skip silently
393 return nil, nil
394 }
395 if resp.StatusCode != 200 {
396 return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
397 }
398
399 var releases []githubRelease
400 if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
401 return nil, err
402 }
403
404 var entries []Entry
405 for _, r := range releases {
406 // Skip drafts and pre-releases
407 if r.Draft || r.Prerelease {
408 continue
409 }
410
411 date := r.CreatedAt
412 if r.PublishedAt != nil {
413 date = *r.PublishedAt
414 }
415
416 // Skip if before since
417 if !since.IsZero() && date.Before(since) {
418 continue
419 }
420
421 tag := r.TagName
422 id := fmt.Sprintf("github-release-%s@%s", repo, tag)
423
424 // Use org as tag
425 var tags []string
426 if parts := strings.SplitN(repo, "/", 2); len(parts) == 2 {
427 org := strings.ToLower(parts[0])
428 if !isGitHubUsername(org) {
429 tags = append(tags, org)
430 }
431 }
432 tags = append(tags, "release")
433
434 entries = append(entries, Entry{
435 ID: id,
436 Kind: KindGitHubRelease,
437 Title: tag,
438 URL: r.HTMLURL,
439 Tags: tags,
440 Date: date,
441 Source: "github",
442 Metadata: map[string]string{
443 "repo": repo,
444 },
445 })
446 }
447
448 return entries, nil
449}
450
451// isGitHubUsername returns true if the label looks like a GitHub username
452// (used as reviewer/maintainer labels in nixpkgs).
453func isGitHubUsername(name string) bool {
454 // Usernames: alphanumeric + hyphens, no spaces, no slashes, typically short
455 if len(name) > 20 || strings.Contains(name, " ") || strings.Contains(name, "/") {
456 return false
457 }
458 // If it starts with uppercase and has no dashes or underscores, likely a username
459 if len(name) > 0 && name[0] >= 'A' && name[0] <= 'Z' &&
460 !strings.Contains(name, "-") && !strings.Contains(name, "_") {
461 // Exception: known non-username capitalized labels
462 switch name {
463 case "NixOS", "Possible":
464 return false
465 }
466 return true
467 }
468 // Explicit known usernames (lowercase)
469 switch name {
470 case "vdemeester", "chmouel", "shortbrain", "illegalstudio",
471 "aliou", "peteonrails", "svkozak":
472 return true
473 }
474 return false
475}