[WIP] Download annotated notebooks

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## rmapi master
 
+* Added geta and mgeta command to read annotated notebooks
+
 * Increased http timeout to 5 minutes to enable upload of larger files
 
 * Add user-agent header to be a good reMarkable citezen

README.md

@@ -27,6 +27,7 @@ Install and build the project:
 
 `go get -u github.com/juruen/rmapi`
 
+
 ## Binary
 
 You can download an already built version for either Linux or OSX from [releases](https://github.com/juruen/rmapi/releases).
@@ -88,6 +89,21 @@ mput /Papers
 
 Use `get path_to_file` to download a file from the cloud to your local computer.
 
+## [optional] Download a file with annotations
+Use `geta path_to_file` to download a file from the cloud to your local computer.
+This command is only available, if all necessary tools are installed:
+ - convert (Note: Policy must be set to read|write for pdfs: https://stackoverflow.com/questions/42928765/convertnot-authorized-aaaa-error-constitute-c-readimage-453)
+ - rsvg-convert
+ - pdf270
+ - pdftk
+ - python3
+
+
+Note: If there exists a template folder under `%HOME/Templates/Remarkable`, 
+then those are used for notebook backgrounds. Otherwise the background is 
+simply white.
+
+
 ## Recursively download directories and files
 
 Use `mget path_to_dir` to recursively download all the files in that directory.
@@ -98,6 +114,18 @@ E.g: download all the files
 mget .
 ```
 
+## [optional] Recursively download directories and files with annotations
+
+Use `mgeta path_to_dir` to recursively download all the annotated files in that directory.
+Note: This command is only available if the geta command is also available.
+
+E.g: download all the files
+
+```
+mgeta .
+```
+
+
 ## Create a directoy
 
 Use `mkdir path_to_new_dir` to create a new directory
@@ -148,3 +176,8 @@ If you want to give it a go, you can run:
 ```bash
 rmapi --fuse-mount mount_point
 ```
+
+
+# Thanks to
+[1] rmapi, https://github.com/juruen/rmapi <br />
+[1] rM2svg, https://github.com/reHackable/maxio <br />

shell/geta.go

@@ -0,0 +1,255 @@
+package shell
+
+import (
+	"errors"
+	"fmt"
+	"time"
+	"os/exec"
+	"strings"
+	"archive/zip"
+	"github.com/juruen/rmapi/log"
+	"os"
+	"io"
+	"github.com/abiosoft/ishell"
+	"path/filepath"
+	"github.com/juruen/rmapi/model"
+)
+
+
+func verifyGetaCmdTools() [] string {
+	var missingTools [] string
+
+	missingTools = verifyCommand("python3", missingTools)
+	missingTools = verifyCommand("convert", missingTools)
+	missingTools = verifyCommand("rsvg-convert", missingTools)
+	missingTools = verifyCommand("pdftk", missingTools)
+	missingTools = verifyCommand("pdf270", missingTools)
+
+	return missingTools
+}
+
+
+// See also https://siongui.github.io/2018/03/16/go-check-if-command-exists/
+func verifyCommand(name string, missingTools [] string) [] string {
+	cmd := exec.Command("/bin/sh", "-c", "command -v "+name)
+	if err := cmd.Run(); err != nil {
+		return append(missingTools, name)
+	}
+	return missingTools
+}
+
+
+func getaCmd(ctx *ShellCtxt) *ishell.Cmd {
+	return &ishell.Cmd{
+		Name:      "geta",
+		Help:      "copy remote file to local with annotations.",
+		Completer: createEntryCompleter(ctx),
+		Func: func(c *ishell.Context) {
+			// Parse cmd args
+			if len(c.Args) == 0 {
+				c.Err(errors.New("missing source file"))
+				return
+			}
+			srcName := c.Args[0]
+
+			// Download document as zip
+			node, err := ctx.api.Filetree.NodeByPath(srcName, ctx.node)
+			if err != nil || node.IsDirectory() {
+				c.Err(errors.New("file doesn't exist"))
+				return
+			}
+
+			err = getAnnotatedDocument(ctx, node, "")
+			if err != nil {
+				c.Err(err)
+				return
+			}
+
+			fmt.Println("OK")
+		},
+	}
+}
+
+
+func getAnnotatedDocument(ctx *ShellCtxt, node *model.Node, path string) error {
+	zipFile := fmt.Sprintf(".%s.zip", node.Name())
+
+	// Set output name and output file name
+	output := node.Name()
+	if(path != ""){
+		output = fmt.Sprintf("%s/%s", path, node.Name())
+	}
+	outputFileName := fmt.Sprintf("%s.pdf", output)
+
+	// Parse last modified time of document on api
+	modifiedClientTime, err := time.Parse(time.RFC3339Nano, node.Document.ModifiedClient)
+	if err != nil {
+		// If we could not parse the time correctly, we still continue 
+		// with the execution such that the pdf is downloaded...
+		log.Warning.Println("Could not parse modified time. Overwrite existing file.")
+		modifiedClientTime = time.Now().Local()
+	}
+
+	// If document has not changed since last update skip pdf convertion
+	outputFile, err := os.Stat(outputFileName)
+	if !os.IsNotExist(err) {
+		outputFileModTime := outputFile.ModTime()
+		if(outputFileModTime.Equal(modifiedClientTime)){
+			log.Trace.Println("Nothing changed since last download. Skip. ")
+			cleanup("", zipFile);
+			return nil
+		}
+	}
+
+	// Download document if content has changed
+	err = ctx.api.FetchDocument(node.Document.ID, zipFile)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Failed to download file %s with %s", node.Name(), err.Error()))
+	}
+
+	// Unzip document
+	tmpFolder := fmt.Sprintf(".%s", node.Document.ID)
+	_, err = unzip(zipFile, tmpFolder)
+	if err != nil {
+		cleanup(tmpFolder, zipFile);
+		return err
+	}
+
+	// Currently we don't support the annotation of epub files
+	if(isEpub(tmpFolder, node)){
+		cleanup(tmpFolder, zipFile);
+		return errors.New("Could not annotate epub files.");
+	}
+
+	// If pdf is annotated convert it, otherwise move the pdf file
+	if(documentIsAnnotated(tmpFolder, node)){
+		exportPdf := os.Getenv("GOPATH") + "/src/github.com/juruen/rmapi/tools/exportAnnotatedPdf"
+		rM2svg := os.Getenv("GOPATH") + "/src/github.com/juruen/rmapi/tools/rM2svg"
+		output, err := exec.Command(
+			"/bin/bash", 
+			exportPdf, 
+			tmpFolder,
+			node.Document.ID, 
+			output, 
+			rM2svg).CombinedOutput()
+
+		log.Trace.Println(fmt.Sprintf("%s", output))
+
+		if err != nil {
+			cleanup(tmpFolder, zipFile);
+			return err
+		}
+
+	} else {
+		log.Trace.Println("Document is not annotated. Move original file.")
+
+		// Two cases exist: Pdf file or a notebook. The second case 
+		// Could not occur because a notebook has always at least one page.
+		// Therefore we know that its a pdf file.
+		plainPdfFile := fmt.Sprintf("%s/%s.pdf", tmpFolder, node.Document.ID)
+		err := os.Rename(plainPdfFile, outputFileName)
+		if err != nil {
+			log.Error.Println("Could not move pdf file.")
+			cleanup(tmpFolder, zipFile);
+			return err
+		}
+	}
+
+	// Set creation time
+	err = os.Chtimes(outputFileName, modifiedClientTime, modifiedClientTime)
+	if err != nil {
+		fmt.Println("(Warning) Could not set modified time of pdf file.")
+	}
+
+	// Cleanup
+	cleanup(tmpFolder, zipFile);
+	return nil
+}
+
+
+func cleanup(tmpFolder string, zipFile string) {
+	_, err := os.Stat(tmpFolder); 
+	if(!os.IsNotExist(err)){
+		os.RemoveAll(tmpFolder)
+	}
+
+	_, err = os.Stat(zipFile);
+	if(!os.IsNotExist(err)){
+		os.Remove(zipFile)
+	}
+}
+
+
+// From https://golangcode.com/unzip-files-in-go/
+func unzip(src string, dest string) ([]string, error) {
+
+    var filenames []string
+
+    r, err := zip.OpenReader(src)
+    if err != nil {
+        return filenames, err
+    }
+    defer r.Close()
+
+    for _, f := range r.File {
+
+        rc, err := f.Open()
+        if err != nil {
+            return filenames, err
+        }
+        defer rc.Close()
+
+        // Store filename/path for returning and using later on
+        fpath := filepath.Join(dest, f.Name)
+
+        // Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
+        if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) {
+            return filenames, fmt.Errorf("%s: illegal file path", fpath)
+        }
+
+        filenames = append(filenames, fpath)
+
+        if f.FileInfo().IsDir() {
+
+            // Make Folder
+            os.MkdirAll(fpath, os.ModePerm)
+
+        } else {
+
+            // Make File
+            if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
+                return filenames, err
+            }
+
+            outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+            if err != nil {
+                return filenames, err
+            }
+
+            _, err = io.Copy(outFile, rc)
+
+            // Close the file without defer to close before next iteration of loop
+            outFile.Close()
+
+            if err != nil {
+                return filenames, err
+            }
+
+        }
+    }
+    return filenames, nil
+}
+
+
+func documentIsAnnotated(tmpFolder string, node *model.Node) bool {
+	annotationFolder := fmt.Sprintf("%s/%s", tmpFolder, node.Document.ID)
+	_, err := os.Stat(annotationFolder); 
+	return !os.IsNotExist(err)
+}
+
+
+func isEpub(tmpFolder string, node *model.Node) bool {
+	annotationFolder := fmt.Sprintf("%s/%s.epub", tmpFolder, node.Document.ID)
+	_, err := os.Stat(annotationFolder); 
+	return !os.IsNotExist(err)
+}

shell/mgeta.go

@@ -0,0 +1,67 @@
+package shell
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/abiosoft/ishell"
+	"github.com/juruen/rmapi/filetree"
+	"github.com/juruen/rmapi/model"
+)
+
+func mgetaCmd(ctx *ShellCtxt) *ishell.Cmd {
+	return &ishell.Cmd{
+		Name:      "mgeta",
+		Help:      "recursively copy remote directory to local with annotations",
+		Completer: createDirCompleter(ctx),
+		Func: func(c *ishell.Context) {
+			if len(c.Args) == 0 {
+				c.Err(errors.New(("missing source dir")))
+				return
+			}
+
+			srcName := c.Args[0]
+
+			node, err := ctx.api.Filetree.NodeByPath(srcName, ctx.node)
+
+			if err != nil || node.IsFile() {
+				c.Err(errors.New("directory doesn't exist"))
+				return
+			}
+
+			visitor := filetree.FileTreeVistor{
+				func(currentNode *model.Node, currentPath []string) bool {
+					idxDir := 0
+					if srcName == "." && len(currentPath) > 0 {
+						idxDir = 1
+					}
+
+					dst := "./" + filetree.BuildPath(currentPath[idxDir:], currentNode.Name())
+					dir := path.Dir(dst)
+					os.MkdirAll(dir, 0766)
+
+					if currentNode.IsDirectory() {
+						return filetree.ContinueVisiting
+					}
+
+					c.Printf("downloading [%s]...", dst)
+
+					err = getAnnotatedDocument(ctx, currentNode, fmt.Sprintf("%s", dir))
+
+					if err == nil {
+						c.Println(" OK")
+						return filetree.ContinueVisiting
+					}
+
+					c.Err(errors.New(fmt.Sprintf("Failed to downlaod file %s", currentNode.Name())))
+
+					return filetree.ContinueVisiting
+				},
+			}
+
+			filetree.WalkTree(node, visitor)
+		},
+	}
+}