diff --git a/README.md b/README.md index 692d3395..36ba515e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,37 @@ To install the par command-line application: go get -u github.com/akalin/gopar/cmd/par ``` -## License +or run + +``` +go build cmd/par/main.go +``` + +after a checkout of this repository. + +### Usage + +Create, verify and repair parity files. Run `par --help` for an overview of all options. + +### libgopar + +Use gopar in your own Go code. Feed a string array with the commandline arguments to Gopar(): + +``` +import "github.com/akalin/gopar/libgopar" +my_args := []strings +libgopar.Gopar(my_args) +``` + +Alternatively, compile a C library: + +``` +go build -buildmode=c-shared -o libgopar.so ./libgopar/C.go +``` + +This library exposes the `gopar()` function. + +### License Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. diff --git a/cmd/par/C.go b/cmd/par/C.go new file mode 100644 index 00000000..95e22039 --- /dev/null +++ b/cmd/par/C.go @@ -0,0 +1,33 @@ +package main + +import ( + "C" + "unsafe" + "fmt" + + "github.com/akalin/gopar/libgopar" +) + +//export gopar +func gopar(argc_ C.int, argv_ **C.char) int { + retval := 7 + defer func() { + if r := recover(); r != nil { + // libgopar only panics with strings currently + fmt.Println("libgopar panicked! ", r) + } + }() + + length := int(argc_) + cStrings := (*[1 << 28]*C.char)(unsafe.Pointer(argv_))[:length:length] + + args_ := make([]string, length+1) + args_[0] = "libgopar" + for i, cString := range cStrings { + args_[i+1] = C.GoString(cString) + } + retval = libgopar.Par(args_,true) + return retval +} + +func main(){} diff --git a/cmd/par/main.go b/cmd/par/main.go index f26fcf29..9b8ca4ef 100644 --- a/cmd/par/main.go +++ b/cmd/par/main.go @@ -1,510 +1,11 @@ package main import ( - "errors" - "flag" - "fmt" - "io/ioutil" "os" - "os/signal" - "path" - "path/filepath" - "runtime/pprof" - "strings" - "github.com/akalin/gopar/par1" - "github.com/akalin/gopar/par2" - "github.com/akalin/gopar/rsec16" + "github.com/akalin/gopar/libgopar" ) -type par1LogEncoderDelegate struct{} - -func (par1LogEncoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, err error) { - if err != nil { - fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err) - } else { - fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount) - } -} - -func (par1LogEncoderDelegate) OnVolumeFileWrite(i, n int, path string, dataByteCount, byteCount int, err error) { - if err != nil { - fmt.Printf("[%d/%d] Writing volume file %q failed: %+v\n", i, n, path, err) - } else { - fmt.Printf("[%d/%d] Wrote volume file %q (%d data bytes, %d bytes)\n", i, n, path, dataByteCount, byteCount) - } -} - -type par1LogDecoderDelegate struct{} - -func (par1LogDecoderDelegate) OnHeaderLoad(headerInfo string) { - fmt.Printf("Loaded header: %s\n", headerInfo) -} - -func (par1LogDecoderDelegate) OnFileEntryLoad(i, n int, filename, entryInfo string) { - fmt.Printf("[%d/%d] Loaded entry for %q: %s\n", i, n, filename, entryInfo) -} - -func (par1LogDecoderDelegate) OnCommentLoad(comment []byte) { - fmt.Printf("Comment: %q\n", comment) -} - -func (par1LogDecoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, corrupt bool, err error) { - if err != nil { - if corrupt { - fmt.Printf("[%d/%d] Loading data file %q failed; marking as corrupt and skipping: %+v\n", i, n, path, err) - } else { - fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err) - } - } else { - fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount) - } -} - -func (par1LogDecoderDelegate) OnDataFileWrite(i, n int, path string, byteCount int, err error) { - if err != nil { - fmt.Printf("[%d/%d] Writing data file %q failed: %+v\n", i, n, path, err) - } else { - fmt.Printf("[%d/%d] Wrote data file %q (%d bytes)\n", i, n, path, byteCount) - } -} - -func (par1LogDecoderDelegate) OnVolumeFileLoad(i uint64, path string, storedSetHash, computedSetHash [16]byte, dataByteCount int, err error) { - if os.IsNotExist(err) { - // Do nothing. - } else if err != nil { - fmt.Printf("[%d] Loading volume file %q failed: %+v\n", i, path, err) - } else { - fmt.Printf("[%d] Loaded volume file %q (%d data bytes)\n", i, path, dataByteCount) - if storedSetHash != computedSetHash { - fmt.Printf("[%d] Warning: stored set hash in %q %x doesn't match computed set hash %x\n", i, path, storedSetHash, computedSetHash) - } - } -} - -type par2LogEncoderDelegate struct{} - -func (par2LogEncoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, err error) { - if err != nil { - fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err) - } else { - fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount) - } -} - -func (par2LogEncoderDelegate) OnIndexFileWrite(path string, byteCount int, err error) { - if err != nil { - fmt.Printf("Writing index file %q failed: %+v\n", path, err) - } else { - fmt.Printf("Wrote index file %q (%d bytes)\n", path, byteCount) - } -} - -func (par2LogEncoderDelegate) OnRecoveryFileWrite(start, count, total int, path string, dataByteCount, byteCount int, err error) { - if err != nil { - fmt.Printf("[%d+%d/%d] Writing recovery file %q failed: %+v\n", start, count, total, path, err) - } else { - fmt.Printf("[%d+%d/%d] Wrote recovery file %q (%d data bytes, %d bytes)\n", start, count, total, path, dataByteCount, byteCount) - } -} - -type par2LogDecoderDelegate struct{} - -func (par2LogDecoderDelegate) OnCreatorPacketLoad(clientID string) { - fmt.Printf("Loaded creator packet with client ID %q\n", clientID) -} - -func (par2LogDecoderDelegate) OnMainPacketLoad(sliceByteCount, recoverySetCount, nonRecoverySetCount int) { - fmt.Printf("Loaded main packet: slice byte count=%d, recovery set size=%d, non-recovery set size=%d\n", sliceByteCount, recoverySetCount, nonRecoverySetCount) -} - -func (par2LogDecoderDelegate) OnFileDescriptionPacketLoad(fileID [16]byte, filename string, byteCount int) { - fmt.Printf("Loaded file description packet for %q (ID=%x, %d bytes)\n", filename, fileID, byteCount) -} - -func (par2LogDecoderDelegate) OnIFSCPacketLoad(fileID [16]byte) { - fmt.Printf("Loaded checksums for file with ID %x\n", fileID) -} - -func (par2LogDecoderDelegate) OnRecoveryPacketLoad(exponent uint16, byteCount int) { - fmt.Printf("Loaded recovery packet: exponent=%d, byte count=%d\n", exponent, byteCount) -} - -func (par2LogDecoderDelegate) OnUnknownPacketLoad(packetType [16]byte, byteCount int) { - fmt.Printf("Loaded unknown packet of type %q and byte count %d\n", packetType, byteCount) -} - -func (par2LogDecoderDelegate) OnOtherPacketSkip(setID [16]byte, packetType [16]byte, byteCount int) { - fmt.Printf("Skipped packet with set ID %x of type %q and byte count %d\n", setID, packetType, byteCount) -} - -func (par2LogDecoderDelegate) OnDataFileLoad(i, n int, path string, byteCount, hits, misses int, err error) { - if err != nil { - fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err) - } else { - fmt.Printf("[%d/%d] Loaded data file %q (%d bytes, %d hits, %d misses)\n", i, n, path, byteCount, hits, misses) - } -} - -func (par2LogDecoderDelegate) OnParityFileLoad(i int, path string, err error) { - if err != nil { - fmt.Printf("[%d] Loading volume file %q failed: %+v\n", i, path, err) - } else { - fmt.Printf("[%d] Loaded volume file %q\n", i, path) - } -} - -func (par2LogDecoderDelegate) OnDetectCorruptDataChunk(fileID [16]byte, filename string, startByteOffset, endByteOffset int) { - fmt.Printf("Corrupt data chunk: %q (ID %x), bytes %d to %d\n", filename, fileID, startByteOffset, endByteOffset-1) -} - -func (par2LogDecoderDelegate) OnDetectDataFileHashMismatch(fileID [16]byte, filename string) { - fmt.Printf("Hash mismatch for %q (ID %x)\n", filename, fileID) -} - -func (par2LogDecoderDelegate) OnDetectDataFileWrongByteCount(fileID [16]byte, filename string) { - fmt.Printf("Wrong byte count for %q (ID %x)\n", filename, fileID) -} - -func (par2LogDecoderDelegate) OnDataFileWrite(i, n int, path string, byteCount int, err error) { - if err != nil { - fmt.Printf("[%d/%d] Writing data file %q failed: %+v\n", i, n, path, err) - } else { - fmt.Printf("[%d/%d] Wrote data file %q (%d bytes)\n", i, n, path, byteCount) - } -} - -func newFlagSet(name string) *flag.FlagSet { - flagSet := flag.NewFlagSet(name, flag.ContinueOnError) - flagSet.SetOutput(ioutil.Discard) - return flagSet -} - -type globalFlags struct { - usage bool - cpuProfile string - numGoroutines int -} - -func getGlobalFlags(name string) (*flag.FlagSet, *globalFlags) { - flagSet := newFlagSet(name) - - var flags globalFlags - flagSet.BoolVar(&flags.usage, "h", false, "print usage info") - flagSet.StringVar(&flags.cpuProfile, "cpuprofile", "", "if non-empty, where to write the CPU profile") - // TODO: Detect hyperthreading and use only number of physical cores. - flagSet.IntVar(&flags.numGoroutines, "g", rsec16.DefaultNumGoroutines(), "number of goroutines to use for encoding/decoding PAR2") - - return flagSet, &flags -} - -type createFlags struct { - sliceByteCount int - numParityShards int -} - -func getCreateFlags(name string) (*flag.FlagSet, *createFlags) { - flagSet := newFlagSet(name + " create") - - var flags createFlags - flagSet.IntVar(&flags.sliceByteCount, "s", 2000, "block size in bytes (must be a multiple of 4)") - flagSet.IntVar(&flags.numParityShards, "c", 3, "number of recovery blocks to create (or files, for PAR1)") - - return flagSet, &flags -} - -type verifyFlags struct { - checkParity bool -} - -func getVerifyFlags(name string) (*flag.FlagSet, *verifyFlags) { - flagSet := newFlagSet(name + " verify") - - var flags verifyFlags - return flagSet, &flags -} - -type repairFlags struct { - checkParity bool -} - -func getRepairFlags(name string) (*flag.FlagSet, *repairFlags) { - flagSet := newFlagSet(name + " repair") - - var flags repairFlags - flagSet.BoolVar(&flags.checkParity, "checkparity", false, "check parity files before repairing") - - return flagSet, &flags -} - -type commandMask int - -const ( - createCommand commandMask = 1 << iota - verifyCommand - repairCommand - allCommands = createCommand | verifyCommand | repairCommand -) - -func printUsageAndExit(name string, mask commandMask, err error) { - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - } - - fmt.Printf("\nUsage:\n") - - if mask&createCommand != 0 { - fmt.Printf(" %s [global options] c(reate) [create options] \n", name) - } - - if mask&verifyCommand != 0 { - fmt.Printf(" %s [global options] v(erify) [verify options] \n", name) - } - - if mask&repairCommand != 0 { - fmt.Printf(" %s [global options] f(epair) [repair options] \n", name) - } - - fmt.Printf("\nGlobal options\n") - globalFlagSet, _ := getGlobalFlags(name) - globalFlagSet.SetOutput(os.Stdout) - globalFlagSet.PrintDefaults() - - if mask&createCommand != 0 { - fmt.Printf("\nCreate options\n") - createFlagSet, _ := getCreateFlags(name) - createFlagSet.SetOutput(os.Stdout) - createFlagSet.PrintDefaults() - } - - if mask&verifyCommand != 0 { - fmt.Printf("\nVerify options\n") - verifyFlagSet, _ := getVerifyFlags(name) - verifyFlagSet.SetOutput(os.Stdout) - verifyFlagSet.PrintDefaults() - } - - if mask&repairCommand != 0 { - fmt.Printf("\nRepair options\n") - repairFlagSet, _ := getRepairFlags(name) - repairFlagSet.SetOutput(os.Stdout) - repairFlagSet.PrintDefaults() - } - - fmt.Printf("\n") - os.Exit(2) -} - -type encoder interface { - LoadFileData() error - ComputeParityData() error - Write(string) error -} - -type decoder interface { - LoadFileData() error - LoadParityData() error - Verify() (needsRepair bool, err error) - Repair(checkParity bool) ([]string, error) -} - -func newEncoder(parFile string, filePaths []string, sliceByteCount, numParityShards, numGoroutines int) (encoder, error) { - // TODO: Detect file type more robustly. - ext := path.Ext(parFile) - if ext == ".par2" { - return par2.NewEncoder(par2LogEncoderDelegate{}, filePaths, sliceByteCount, numParityShards, numGoroutines) - } - return par1.NewEncoder(par1LogEncoderDelegate{}, filePaths, numParityShards) -} - -func newDecoder(parFile string, numGoroutines int) (decoder, error) { - // TODO: Detect file type more robustly. - ext := path.Ext(parFile) - if ext == ".par2" { - return par2.NewDecoder(par2LogDecoderDelegate{}, parFile, numGoroutines) - } - return par1.NewDecoder(par1LogDecoderDelegate{}, parFile) -} - -// Taken from https://github.com/brenthuisman/libpar2/blob/master/src/libpar2.h#L109 . -const ( - eSuccess = 0 - eRepairPossible = 1 - eRepairNotPossible = 2 - eInvalidCommandLineArguments = 3 - eInsufficientCriticalData = 4 - eRepairFailed = 5 - eFileIOError = 6 - eLogicError = 7 - eMemoryError = 8 -) - -func processVerifyOrRepairError(needsRepair bool, err error) { - // Match exit codes to par2cmdline. - if err != nil { - switch err.(type) { - case rsec16.NotEnoughParityShardsError: - fmt.Fprintf(os.Stderr, "Repair necessary but not possible.\n") - os.Exit(eRepairNotPossible) - default: - fmt.Fprintf(os.Stderr, "Error encountered: %s\n", err) - os.Exit(eLogicError) - } - } - if needsRepair { - fmt.Fprintf(os.Stderr, "Repair necessary and possible.\n") - os.Exit(eRepairPossible) - } - os.Exit(eSuccess) -} - func main() { - name := filepath.Base(os.Args[0]) - - globalFlagSet, globalFlags := getGlobalFlags(name) - err := globalFlagSet.Parse(os.Args[1:]) - if err == nil && globalFlagSet.NArg() == 0 { - err = errors.New("no command specified") - } - if err != nil || globalFlags.usage { - printUsageAndExit(name, allCommands, err) - } - - if globalFlags.cpuProfile != "" { - f, err := os.Create(globalFlags.cpuProfile) - if err != nil { - panic(err) - } - defer f.Close() - - err = pprof.StartCPUProfile(f) - if err != nil { - panic(err) - } - - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - go func() { - <-c - pprof.StopCPUProfile() - os.Exit(1) - }() - - defer pprof.StopCPUProfile() - } - - cmd := globalFlagSet.Arg(0) - args := globalFlagSet.Args()[1:] - - switch strings.ToLower(cmd) { - case "c": - fallthrough - case "create": - createFlagSet, createFlags := getCreateFlags(name) - err := createFlagSet.Parse(args) - if err == nil { - if createFlagSet.NArg() == 0 { - err = errors.New("no PAR file specified") - } else if createFlagSet.NArg() == 1 { - err = errors.New("no data files specified") - } - } - if err != nil { - printUsageAndExit(name, createCommand, err) - } - - parFile := createFlagSet.Arg(0) - files := createFlagSet.Args()[1:] - - encoder, err := newEncoder(parFile, files, createFlags.sliceByteCount, createFlags.numParityShards, globalFlags.numGoroutines) - if err != nil { - panic(err) - } - - err = encoder.LoadFileData() - if err != nil { - panic(err) - } - - err = encoder.ComputeParityData() - if err != nil { - panic(err) - } - - err = encoder.Write(parFile) - if err != nil { - fmt.Printf("Write parity error: %s", err) - os.Exit(-1) - } - - case "v": - fallthrough - case "verify": - verifyFlagSet, _ := getVerifyFlags(name) - err := verifyFlagSet.Parse(args) - if err == nil && verifyFlagSet.NArg() == 0 { - err = errors.New("no PAR file specified") - } - if err != nil { - printUsageAndExit(name, verifyCommand, err) - } - - parFile := verifyFlagSet.Arg(0) - - decoder, err := newDecoder(parFile, globalFlags.numGoroutines) - if err != nil { - panic(err) - } - - err = decoder.LoadFileData() - if err != nil { - panic(err) - } - - err = decoder.LoadParityData() - if err != nil { - panic(err) - } - - needsRepair, err := decoder.Verify() - processVerifyOrRepairError(needsRepair, err) - - case "r": - fallthrough - case "repair": - repairFlagSet, repairFlags := getRepairFlags(name) - err := repairFlagSet.Parse(args) - if err == nil && repairFlagSet.NArg() == 0 { - err = errors.New("no PAR file specified") - } - if err != nil { - printUsageAndExit(name, repairCommand, err) - } - - parFile := repairFlagSet.Arg(0) - - decoder, err := newDecoder(parFile, globalFlags.numGoroutines) - if err != nil { - panic(err) - } - - err = decoder.LoadFileData() - if err != nil { - panic(err) - } - - err = decoder.LoadParityData() - if err != nil { - panic(err) - } - - repairedFiles, err := decoder.Repair(repairFlags.checkParity) - fmt.Printf("Repaired files: %v\n", repairedFiles) - needsRepair := false - processVerifyOrRepairError(needsRepair, err) - - default: - err := fmt.Errorf("unknown command '%s'", cmd) - printUsageAndExit(name, allCommands, err) - } -} + os.Exit(libgopar.Par(os.Args,false)) +} \ No newline at end of file diff --git a/libgopar/gopar.go b/libgopar/gopar.go new file mode 100644 index 00000000..b4ce4931 --- /dev/null +++ b/libgopar/gopar.go @@ -0,0 +1,523 @@ +package libgopar + +import ( + "errors" + "flag" + "fmt" + "io/ioutil" + "os" + "os/signal" + "path" + "path/filepath" + "runtime/pprof" + "strings" + + "github.com/akalin/gopar/par1" + "github.com/akalin/gopar/par2" + "github.com/akalin/gopar/rsec16" +) + +type par1LogEncoderDelegate struct{} + +func (par1LogEncoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, err error) { + if err != nil { + fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err) + } else { + fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount) + } +} + +func (par1LogEncoderDelegate) OnVolumeFileWrite(i, n int, path string, dataByteCount, byteCount int, err error) { + if err != nil { + fmt.Printf("[%d/%d] Writing volume file %q failed: %+v\n", i, n, path, err) + } else { + fmt.Printf("[%d/%d] Wrote volume file %q (%d data bytes, %d bytes)\n", i, n, path, dataByteCount, byteCount) + } +} + +type par1LogDecoderDelegate struct{} + +func (par1LogDecoderDelegate) OnHeaderLoad(headerInfo string) { + fmt.Printf("Loaded header: %s\n", headerInfo) +} + +func (par1LogDecoderDelegate) OnFileEntryLoad(i, n int, filename, entryInfo string) { + fmt.Printf("[%d/%d] Loaded entry for %q: %s\n", i, n, filename, entryInfo) +} + +func (par1LogDecoderDelegate) OnCommentLoad(comment []byte) { + fmt.Printf("Comment: %q\n", comment) +} + +func (par1LogDecoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, corrupt bool, err error) { + if err != nil { + if corrupt { + fmt.Printf("[%d/%d] Loading data file %q failed; marking as corrupt and skipping: %+v\n", i, n, path, err) + } else { + fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err) + } + } else { + fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount) + } +} + +func (par1LogDecoderDelegate) OnDataFileWrite(i, n int, path string, byteCount int, err error) { + if err != nil { + fmt.Printf("[%d/%d] Writing data file %q failed: %+v\n", i, n, path, err) + } else { + fmt.Printf("[%d/%d] Wrote data file %q (%d bytes)\n", i, n, path, byteCount) + } +} + +func (par1LogDecoderDelegate) OnVolumeFileLoad(i uint64, path string, storedSetHash, computedSetHash [16]byte, dataByteCount int, err error) { + if os.IsNotExist(err) { + // Do nothing. + } else if err != nil { + fmt.Printf("[%d] Loading volume file %q failed: %+v\n", i, path, err) + } else { + fmt.Printf("[%d] Loaded volume file %q (%d data bytes)\n", i, path, dataByteCount) + if storedSetHash != computedSetHash { + fmt.Printf("[%d] Warning: stored set hash in %q %x doesn't match computed set hash %x\n", i, path, storedSetHash, computedSetHash) + } + } +} + +type par2LogEncoderDelegate struct{} + +func (par2LogEncoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, err error) { + if err != nil { + fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err) + } else { + fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount) + } +} + +func (par2LogEncoderDelegate) OnIndexFileWrite(path string, byteCount int, err error) { + if err != nil { + fmt.Printf("Writing index file %q failed: %+v\n", path, err) + } else { + fmt.Printf("Wrote index file %q (%d bytes)\n", path, byteCount) + } +} + +func (par2LogEncoderDelegate) OnRecoveryFileWrite(start, count, total int, path string, dataByteCount, byteCount int, err error) { + if err != nil { + fmt.Printf("[%d+%d/%d] Writing recovery file %q failed: %+v\n", start, count, total, path, err) + } else { + fmt.Printf("[%d+%d/%d] Wrote recovery file %q (%d data bytes, %d bytes)\n", start, count, total, path, dataByteCount, byteCount) + } +} + +type par2LogDecoderDelegate struct{} + +func (par2LogDecoderDelegate) OnCreatorPacketLoad(clientID string) { + fmt.Printf("Loaded creator packet with client ID %q\n", clientID) +} + +func (par2LogDecoderDelegate) OnMainPacketLoad(sliceByteCount, recoverySetCount, nonRecoverySetCount int) { + fmt.Printf("Loaded main packet: slice byte count=%d, recovery set size=%d, non-recovery set size=%d\n", sliceByteCount, recoverySetCount, nonRecoverySetCount) +} + +func (par2LogDecoderDelegate) OnFileDescriptionPacketLoad(fileID [16]byte, filename string, byteCount int) { + fmt.Printf("Loaded file description packet for %q (ID=%x, %d bytes)\n", filename, fileID, byteCount) +} + +func (par2LogDecoderDelegate) OnIFSCPacketLoad(fileID [16]byte) { + fmt.Printf("Loaded checksums for file with ID %x\n", fileID) +} + +func (par2LogDecoderDelegate) OnRecoveryPacketLoad(exponent uint16, byteCount int) { + fmt.Printf("Loaded recovery packet: exponent=%d, byte count=%d\n", exponent, byteCount) +} + +func (par2LogDecoderDelegate) OnUnknownPacketLoad(packetType [16]byte, byteCount int) { + fmt.Printf("Loaded unknown packet of type %q and byte count %d\n", packetType, byteCount) +} + +func (par2LogDecoderDelegate) OnOtherPacketSkip(setID [16]byte, packetType [16]byte, byteCount int) { + fmt.Printf("Skipped packet with set ID %x of type %q and byte count %d\n", setID, packetType, byteCount) +} + +func (par2LogDecoderDelegate) OnDataFileLoad(i, n int, path string, byteCount, hits, misses int, err error) { + if err != nil { + fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err) + } else { + fmt.Printf("[%d/%d] Loaded data file %q (%d bytes, %d hits, %d misses)\n", i, n, path, byteCount, hits, misses) + } +} + +func (par2LogDecoderDelegate) OnParityFileLoad(i int, path string, err error) { + if err != nil { + fmt.Printf("[%d] Loading volume file %q failed: %+v\n", i, path, err) + } else { + fmt.Printf("[%d] Loaded volume file %q\n", i, path) + } +} + +func (par2LogDecoderDelegate) OnDetectCorruptDataChunk(fileID [16]byte, filename string, startByteOffset, endByteOffset int) { + fmt.Printf("Corrupt data chunk: %q (ID %x), bytes %d to %d\n", filename, fileID, startByteOffset, endByteOffset-1) +} + +func (par2LogDecoderDelegate) OnDetectDataFileHashMismatch(fileID [16]byte, filename string) { + fmt.Printf("Hash mismatch for %q (ID %x)\n", filename, fileID) +} + +func (par2LogDecoderDelegate) OnDetectDataFileWrongByteCount(fileID [16]byte, filename string) { + fmt.Printf("Wrong byte count for %q (ID %x)\n", filename, fileID) +} + +func (par2LogDecoderDelegate) OnDataFileWrite(i, n int, path string, byteCount int, err error) { + if err != nil { + fmt.Printf("[%d/%d] Writing data file %q failed: %+v\n", i, n, path, err) + } else { + fmt.Printf("[%d/%d] Wrote data file %q (%d bytes)\n", i, n, path, byteCount) + } +} + +func newFlagSet(name string) *flag.FlagSet { + flagSet := flag.NewFlagSet(name, flag.ContinueOnError) + flagSet.SetOutput(ioutil.Discard) + return flagSet +} + +type globalFlags struct { + usage bool + cpuProfile string + numGoroutines int +} + +func getGlobalFlags(name string) (*flag.FlagSet, *globalFlags) { + flagSet := newFlagSet(name) + + var flags globalFlags + flagSet.BoolVar(&flags.usage, "h", false, "print usage info") + flagSet.StringVar(&flags.cpuProfile, "cpuprofile", "", "if non-empty, where to write the CPU profile") + // TODO: Detect hyperthreading and use only number of physical cores. + flagSet.IntVar(&flags.numGoroutines, "g", rsec16.DefaultNumGoroutines(), "number of goroutines to use for encoding/decoding PAR2") + + return flagSet, &flags +} + +type createFlags struct { + sliceByteCount int + numParityShards int +} + +func getCreateFlags(name string) (*flag.FlagSet, *createFlags) { + flagSet := newFlagSet(name + " create") + + var flags createFlags + flagSet.IntVar(&flags.sliceByteCount, "s", 2000, "block size in bytes (must be a multiple of 4)") + flagSet.IntVar(&flags.numParityShards, "c", 3, "number of recovery blocks to create (or files, for PAR1)") + + return flagSet, &flags +} + +type verifyFlags struct { + checkParity bool +} + +func getVerifyFlags(name string) (*flag.FlagSet, *verifyFlags) { + flagSet := newFlagSet(name + " verify") + + var flags verifyFlags + return flagSet, &flags +} + +type repairFlags struct { + checkParity bool +} + +func getRepairFlags(name string) (*flag.FlagSet, *repairFlags) { + flagSet := newFlagSet(name + " repair") + + var flags repairFlags + flagSet.BoolVar(&flags.checkParity, "checkparity", false, "check parity files before repairing") + + return flagSet, &flags +} + +type commandMask int + +const ( + createCommand commandMask = 1 << iota + verifyCommand + repairCommand + allCommands = createCommand | verifyCommand | repairCommand +) + +func printUsage(name string, mask commandMask, err error) { + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + } + + fmt.Printf("\nUsage:\n") + + if mask&createCommand != 0 { + fmt.Printf(" %s [global options] c(reate) [create options] \n", name) + } + + if mask&verifyCommand != 0 { + fmt.Printf(" %s [global options] v(erify) [verify options] \n", name) + } + + if mask&repairCommand != 0 { + fmt.Printf(" %s [global options] f(epair) [repair options] \n", name) + } + + fmt.Printf("\nGlobal options\n") + globalFlagSet, _ := getGlobalFlags(name) + globalFlagSet.SetOutput(os.Stdout) + globalFlagSet.PrintDefaults() + + if mask&createCommand != 0 { + fmt.Printf("\nCreate options\n") + createFlagSet, _ := getCreateFlags(name) + createFlagSet.SetOutput(os.Stdout) + createFlagSet.PrintDefaults() + } + + if mask&verifyCommand != 0 { + fmt.Printf("\nVerify options\n") + verifyFlagSet, _ := getVerifyFlags(name) + verifyFlagSet.SetOutput(os.Stdout) + verifyFlagSet.PrintDefaults() + } + + if mask&repairCommand != 0 { + fmt.Printf("\nRepair options\n") + repairFlagSet, _ := getRepairFlags(name) + repairFlagSet.SetOutput(os.Stdout) + repairFlagSet.PrintDefaults() + } + + fmt.Printf("\n") +} + +type encoder interface { + LoadFileData() error + ComputeParityData() error + Write(string) error +} + +type decoder interface { + LoadFileData() error + LoadParityData() error + Verify() (needsRepair bool, err error) + Repair(checkParity bool) ([]string, error) +} + +func newEncoder(parFile string, filePaths []string, sliceByteCount, numParityShards, numGoroutines int) (encoder, error) { + // TODO: Detect file type more robustly. + ext := path.Ext(parFile) + if ext == ".par2" { + return par2.NewEncoder(par2LogEncoderDelegate{}, filePaths, sliceByteCount, numParityShards, numGoroutines) + } + return par1.NewEncoder(par1LogEncoderDelegate{}, filePaths, numParityShards) +} + +func newDecoder(parFile string, numGoroutines int) (decoder, error) { + // TODO: Detect file type more robustly. + ext := path.Ext(parFile) + if ext == ".par2" { + return par2.NewDecoder(par2LogDecoderDelegate{}, parFile, numGoroutines) + } + return par1.NewDecoder(par1LogDecoderDelegate{}, parFile) +} + +// Taken from https://github.com/brenthuisman/libpar2/blob/master/src/libpar2.h#L109 . +const ( + eSuccess = 0 + eRepairPossible = 1 + eRepairNotPossible = 2 + eInvalidCommandLineArguments = 3 + eInsufficientCriticalData = 4 + eRepairFailed = 5 + eFileIOError = 6 + eLogicError = 7 + eMemoryError = 8 +) + +func processVerifyOrRepairError(needsRepair bool, err error) int { + // Match exit codes to par2cmdline. + if err != nil { + switch err.(type) { + case rsec16.NotEnoughParityShardsError: + fmt.Fprintf(os.Stderr, "Repair necessary but not possible.\n") + return eRepairNotPossible + default: + fmt.Fprintf(os.Stderr, "Error encountered: %s\n", err) + return eLogicError + } + } + if needsRepair { + fmt.Fprintf(os.Stderr, "Repair necessary and possible.\n") + return eRepairPossible + } + return eSuccess +} + +func Par(args_ []string, islib bool) int { + name := filepath.Base(args_[0]) + + globalFlagSet, globalFlags := getGlobalFlags(name) + err := globalFlagSet.Parse(args_[1:]) + if err == nil && globalFlagSet.NArg() == 0 { + err = errors.New("no command specified") + } + if err != nil || globalFlags.usage { + printUsage(name, allCommands, err) + return eSuccess + } + + if islib == false && globalFlags.cpuProfile != "" { + f, err := os.Create(globalFlags.cpuProfile) + if err != nil { + panic(err) + } + defer f.Close() + + err = pprof.StartCPUProfile(f) + if err != nil { + panic(err) + } + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + <-c + pprof.StopCPUProfile() + os.Exit(eMemoryError) + // return eMemoryError + }() + + defer pprof.StopCPUProfile() + } + + cmd := globalFlagSet.Arg(0) + args := globalFlagSet.Args()[1:] + + switch strings.ToLower(cmd) { + case "c": + fallthrough + case "create": + createFlagSet, createFlags := getCreateFlags(name) + err := createFlagSet.Parse(args) + if err == nil { + if createFlagSet.NArg() == 0 { + err = errors.New("no PAR file specified") + } else if createFlagSet.NArg() == 1 { + err = errors.New("no data files specified") + } + } + if err != nil { + printUsage(name, createCommand, err) + return eInvalidCommandLineArguments + } + + parFile := createFlagSet.Arg(0) + files := createFlagSet.Args()[1:] + + encoder, err := newEncoder(parFile, files, createFlags.sliceByteCount, createFlags.numParityShards, globalFlags.numGoroutines) + if err != nil { + panic(err) + } + + err = encoder.LoadFileData() + if err != nil { + panic(err) + } + + err = encoder.ComputeParityData() + if err != nil { + panic(err) + } + + err = encoder.Write(parFile) + if err != nil { + fmt.Printf("Write parity error: %s", err) + return eFileIOError + } + return eSuccess + + case "v": + fallthrough + case "verify": + verifyFlagSet, _ := getVerifyFlags(name) + err := verifyFlagSet.Parse(args) + if err == nil && verifyFlagSet.NArg() == 0 { + err = errors.New("no PAR file specified") + } + if err != nil { + printUsage(name, verifyCommand, err) + return eInvalidCommandLineArguments + } + + parFile := verifyFlagSet.Arg(0) + + decoder, err := newDecoder(parFile, globalFlags.numGoroutines) + if err != nil { + panic(err) + } + + err = decoder.LoadFileData() + if err != nil { + panic(err) + } + + err = decoder.LoadParityData() + if err != nil { + panic(err) + } + + needsRepair, err := decoder.Verify() + return processVerifyOrRepairError(needsRepair, err) + + case "r": + fallthrough + case "repair": + repairFlagSet, repairFlags := getRepairFlags(name) + err := repairFlagSet.Parse(args) + if err == nil && repairFlagSet.NArg() == 0 { + err = errors.New("no PAR file specified") + } + if err != nil { + printUsage(name, repairCommand, err) + return eInvalidCommandLineArguments + } + + parFile := repairFlagSet.Arg(0) + + decoder, err := newDecoder(parFile, globalFlags.numGoroutines) + if err != nil { + panic(err) + } + + err = decoder.LoadFileData() + if err != nil { + panic(err) + } + + err = decoder.LoadParityData() + if err != nil { + panic(err) + } + + repairedFiles, err := decoder.Repair(repairFlags.checkParity) + fmt.Printf("Repaired files: %v\n", repairedFiles) + needsRepair := false + return processVerifyOrRepairError(needsRepair, err) + + default: + err := fmt.Errorf("unknown command '%s'", cmd) + printUsage(name, allCommands, err) + return eInvalidCommandLineArguments + } + + return eInvalidCommandLineArguments +} + +func Gopar(args_ []string) int { + args_ = append([]string{"libgopar"}, args_...) + return (Par(args_,true)) +}