Commit b69709e7f3cf

@@ -1,1 +1,12 @@
 package org
+
+/*
+For each results:
+- Define a filename (denote naming — gonna be weird but meh)
+- Detect if the file exists
+- If the file doesn't exist, create the file
+- If the file exist, append
+
+For the file format: org file with denote naming
+And use the update date to add new highlights
+*/

@@ -1,30 +1,61 @@
 package readwise
 
-// TODO: support pages
-
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
+	"strings"
 	"time"
 )
 
-func FetchFromAPI(ctx context.Context, apikey string, updateAfter *time.Time) (Export, error) {
-	export := Export{}
-	endpoint := "https://readwise.io/api/v2/export"
-	if updateAfter != nil {
-		endpoint = endpoint + "/?updateAfter=" + updateAfter.Format(time.RFC3339)
-	}
+const (
+	exportEndpoint     = "https://readwise.io/api/v2/export/?"
+	FormatUpdatedAfter = "2006-01-02T15:04:05"
+)
 
+func FetchFromAPI(ctx context.Context, apikey string, updateAfter *time.Time) ([]Result, error) {
+	results := []Result{}
 	httpClient := &http.Client{}
 
+	var e Export
+	var err error
+	var nextPageCursor *int = nil
+	for {
+		e, err = fetchExport(ctx, httpClient, apikey, updateAfter, nextPageCursor)
+		if err != nil {
+			return results, err
+		}
+		results = append(results, e.Results...)
+		nextPageCursor = e.NextPageCursor
+		if nextPageCursor == nil {
+			// No more pages to fetch, we get out
+			break
+		}
+	}
+
+	return results, nil
+}
+
+func fetchExport(ctx context.Context, client *http.Client, apikey string, updateAfter *time.Time, nextPageCursor *int) (Export, error) {
+	export := Export{}
+	endpoint := exportEndpoint
+	params := []string{}
+	if updateAfter != nil {
+		params = append(params, "updatedAfter="+updateAfter.Format(FormatUpdatedAfter))
+	}
+	if nextPageCursor != nil {
+		params = append(params, fmt.Sprintf("pageCursor=%d", *nextPageCursor))
+	}
+	endpoint = endpoint + strings.Join(params, "&&")
+	fmt.Println(endpoint)
 	req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
 	if err != nil {
 		return export, err
 	}
 	req.Header.Add("Authorization", "Token "+apikey)
-	resp, err := httpClient.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		return export, err
 	}

@@ -2,38 +2,79 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
+	"log"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/vdemeester/home/tools/go-org-readwise/internal/readwise"
 )
 
 func main() {
-	ctx := context.Background()
-	highlights, merr := readwise.FetchFromAPI(ctx, os.Getenv("READWISE_KEY"), nil)
-	if merr != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", merr)
-		os.Exit(1)
-	}
-	fmt.Println("count", highlights.Count)
-	fmt.Println("nextPageCursor", *highlights.NextPageCursor)
-	fmt.Println("size", len(highlights.Results))
+	apiKeyFile := flag.String("apiKeyFile", "", "File to load the apiKey from. If empty, it will defer to the READWISE_KEY environment variable")
+	targetFolder := flag.String("targetFolder", "", "Folder to write highlights (in org file) into")
+	flag.Parse()
 
-	updateAfter := time.Now().Add(-1000 * time.Hour)
-	fmt.Println("updateAfter:", updateAfter)
-	highlights, merr = readwise.FetchFromAPI(ctx, os.Getenv("READWISE_KEY"), &updateAfter)
-	if merr != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", merr)
-		os.Exit(1)
+	if *targetFolder == "" {
+		log.Fatal("-targetFolder is a required flag")
 	}
-	fmt.Println("count", highlights.Count)
-	fmt.Println("nextPageCursor", *highlights.NextPageCursor)
-	fmt.Println("size", len(highlights.Results))
-	for _, h := range highlights.Results {
-		fmt.Println("title", h.Title, len(h.Highlights), h.BookTags)
-		// for _, hh := range h.Highlights {
-		// 	fmt.Println(">>>", hh.ID, hh.Tags)
-		// }
+
+	apiKeyData, err := os.ReadFile(*apiKeyFile)
+	if err != nil && !os.IsNotExist(err) {
+		log.Fatalf("Error reading apiKeyFile %s: %v", *apiKeyFile, err)
 	}
+	apikey := string(apiKeyData)
+	if apikey == "" {
+		apikey = os.Getenv("READWISE_KEY")
+	}
+
+	stateFile := filepath.Join(*targetFolder, ".readwise-sync.state")
+	updateAfter, err := getUpdateAfterFromFile(stateFile)
+	if err != nil {
+		log.Fatalf("Error reading readwise state file from %s: %v", stateFile, err)
+	}
+	fmt.Println(*targetFolder)
+	fmt.Println("updateAfter", updateAfter)
+	ctx := context.Background()
+	highlights, err := readwise.FetchFromAPI(ctx, apikey, updateAfter)
+	if err != nil {
+		log.Fatalf("Error while fetching highlights: %v", err)
+	}
+	// if err := os.WriteFile(stateFile, []byte(time.Now().Format(readwise.FormatUpdatedAfter)), 0o666); err != nil {
+	// 	log.Fatalf("Error writing readwise state file in %s: %v", stateFile, err)
+	// }
+	fmt.Println("size", len(highlights))
+
+	// updateAfter := time.Now().Add(-72 * time.Hour)
+	// fmt.Println("updateAfter:", updateAfter)
+	// mhighlights, merr := readwise.FetchFromAPI(ctx, os.Getenv("READWISE_KEY"), &updateAfter)
+	// if merr != nil {
+	// 	fmt.Fprintf(os.Stderr, "%v\n", merr)
+	// 	os.Exit(1)
+	// }
+	// fmt.Println("size", len(mhighlights))
+	// for _, h := range highlights {
+	// 	fmt.Println("title", h.Title, len(h.Highlights), h.BookTags)
+	// 	// for _, hh := range h.Highlights {
+	// 	// 	fmt.Println(">>>", hh.ID, hh.Tags)
+	// 	// }
+	// }
+}
+
+func getUpdateAfterFromFile(stateFile string) (*time.Time, error) {
+	data, err := os.ReadFile(stateFile)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	// If the file doesn't exists, do not fail
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	t, err := time.Parse(readwise.FormatUpdatedAfter, string(data))
+	if err != nil {
+		return nil, err
+	}
+	return &t, nil
 }