Initial commit - some small parts working

2025-10-30 23:17:59 +02:00 · 2012-11-18 17:32:31 +00:00
commit e9ae4f89a4
6 changed files with 377 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,5 @@
+*~
+*.pyc
+test-env*
+junk/
+swiftsync
--- a/20
+++ b/20
@@ -0,0 +1,20 @@
+Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
--- a/README.md
+++ b/README.md
@@ -0,0 +1,53 @@
+Swiftsync
+==========
+
+Sync files and directories to and from swift
+
+FIXME
+
+
+Install
+-------
+
+Swiftsync is a Go program and comes as a single binary file.
+
+Download the relevant binary from
+
+- http://www.craig-wood.com/nick/pub/swiftsync/
+
+Or alternatively if you have Go installed use
+
+    go get github.com/ncw/swiftsync
+
+and this will build the binary in `$GOPATH/bin`.  You can then modify
+the source and submit patches.
+
+Usage
+-----
+
+FIXME
+
+License
+-------
+
+This is free software under the terms of MIT the license (check the
+COPYING file included in this package).
+
+Contact and support
+-------------------
+
+The project website is at:
+
+- https://github.com/ncw/swiftsync
+
+There you can file bug reports, ask for help or contribute patches.
+
+Authors
+-------
+
+- Nick Craig-Wood <nick@craig-wood.com>
+
+Contributors
+------------
+
+- Your name goes here!
--- a/notes.txt
+++ b/notes.txt
@@ -0,0 +1,32 @@
+make 100% compatible with swift.py?
+
+Make Env vars compatible with st?
+
+Get and put the metadata in the libray (x-object-meta-mtime) when getting and putting a file?
+
+st is setting this
+'x-object-meta-mtime'
+
+    getmtime(filename)
+        Return the last modification time of a file, reported by os.stat().
+
+>>> f = os.path.getmtime("z")
+1347717491.343554
+>>> print f
+1347717491.34
+>>> str(f)
+'1347717491.34'
+>>> "%d" % f
+'1347717491'
+>>> 
+
+swift.py appears to be doing it wrong with str(float) which isn't a
+good way of stringifying floats...
+
+Make 
+
+This also puts meta-mtime
+https://github.com/gholt/swiftly
+
+As an integer, but it does parse it as a float
+subargs.append('x-object-meta-mtime:%d' % getmtime(options.input_))
--- a/swiftsync.go
+++ b/swiftsync.go
@@ -0,0 +1,155 @@
+// Sync files and directories to and from swift
+// 
+// Nick Craig-Wood <nick@craig-wood.com>
+package main
+
+import (
+	//"bytes"
+	"flag"
+	"fmt"
+	//"io"
+	//"io/ioutil"
+	"log"
+	//"math/rand"
+	"os"
+	//"os/signal"
+	//"path/filepath"
+	//"regexp"
+	//"runtime"
+	"runtime/pprof"
+	"strconv"
+	"strings"
+	//"sync"
+	//"syscall"
+	//"time"
+	"github.com/ncw/swift"
+)
+
+// Globals
+var (
+	// Flags
+	//fileSize      = flag.Int64("s", 1E9, "Size of the check files")
+	cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
+	//duration      = flag.Duration("duration", time.Hour*24, "Duration to run test")
+	//statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
+	//logfile       = flag.String("logfile", "stressdisk.log", "File to write log to set to empty to ignore")
+
+	snet    = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
+	verbose = flag.Bool("verbose", false, "Print lots more stuff")
+	quiet   = flag.Bool("quiet", false, "Print as little stuff as possible")
+	// FIXME make these part of swift so we get a standard set of flags?
+	authUrl  = flag.String("auth", os.Getenv("SWIFT_AUTH_USER"), "Auth URL for server. Defaults to environment var SWIFT_AUTH_USER.")
+	userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
+	apiKey   = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
+)
+
+// Turns a number of ns into a floating point string in seconds
+//
+// Trims trailing zeros and guaranteed to be perfectly accurate
+func nsToFloatString(ns int64) string {
+	if ns < 0 {
+		return "-" + nsToFloatString(-ns)
+	}
+	result := fmt.Sprintf("%010d", ns)
+	split := len(result) - 9
+	result, decimals := result[:split], result[split:]
+	decimals = strings.TrimRight(decimals, "0")
+	if decimals != "" {
+		result += "."
+		result += decimals
+	}
+	return result
+}
+
+// Turns a floating point string in seconds into a ns integer
+//
+// Guaranteed to be perfectly accurate
+func floatStringToNs(s string) (ns int64, err error) {
+	if s != "" && s[0] == '-' {
+		ns, err = floatStringToNs(s[1:])
+		return -ns, err
+	}
+	point := strings.IndexRune(s, '.')
+	if point >= 0 {
+		tail := s[point+1:]
+		if len(tail) > 0 {
+			if len(tail) > 9 {
+				tail = tail[:9]
+			}
+			uns, err := strconv.ParseUint(tail, 10, 64)
+			if err != nil {
+				return 0, err
+			}
+			ns = int64(uns)
+			for i := 9 - len(tail); i > 0; i-- {
+				ns *= 10
+			}
+		}
+		s = s[:point]
+	}
+	secs, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	ns += int64(1000000000) * secs
+	return ns, nil
+}
+
+// syntaxError prints the syntax
+func syntaxError() {
+	fmt.Fprintf(os.Stderr, `Sync files and directores to and from swift
+
+FIXME
+
+Full options:
+`)
+	flag.PrintDefaults()
+}
+
+// Exit with the message
+func fatal(message string, args ...interface{}) {
+	syntaxError()
+	fmt.Fprintf(os.Stderr, message, args...)
+	os.Exit(1)
+}
+
+func main() {
+	flag.Usage = syntaxError
+	flag.Parse()
+	//args := flag.Args()
+	//runtime.GOMAXPROCS(3)
+
+	// Setup profiling if desired
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+
+	// if len(args) < 1 {
+	// 	fatal("No command supplied\n")
+	// }
+
+	if *userName == "" {
+		log.Fatal("Need --user or environmental variable ST_USER")
+	}
+	if *apiKey == "" {
+		log.Fatal("Need --key or environmental variable ST_KEY")
+	}
+	if *authUrl == "" {
+		log.Fatal("Need --auth or environmental variable ST_AUTH")
+	}
+	c := swift.Connection{
+		UserName: *userName,
+		ApiKey:   *apiKey,
+		AuthUrl:  *authUrl,
+	}
+	err := c.Authenticate()
+	if err != nil {
+		log.Fatal("Failed to authenticate", err)
+	}
+
+}
--- a/swiftsync_test.go
+++ b/swiftsync_test.go
@@ -0,0 +1,112 @@
+// Tests for swiftsync
+package main
+
+import (
+	"testing"
+)
+
+func TestNsToFloatString(t *testing.T) {
+	for _, d := range []struct {
+		ns int64
+		fs string
+	}{
+		{0, "0"},
+		{1, "0.000000001"},
+		{1000, "0.000001"},
+		{1000000, "0.001"},
+		{100000000, "0.1"},
+		{1000000000, "1"},
+		{10000000000, "10"},
+		{12345678912, "12.345678912"},
+		{12345678910, "12.34567891"},
+		{12345678900, "12.3456789"},
+		{12345678000, "12.345678"},
+		{12345670000, "12.34567"},
+		{12345600000, "12.3456"},
+		{12345000000, "12.345"},
+		{12340000000, "12.34"},
+		{12300000000, "12.3"},
+		{12000000000, "12"},
+		{10000000000, "10"},
+		{1347717491123123123, "1347717491.123123123"},
+	} {
+		if nsToFloatString(d.ns) != d.fs {
+			t.Error("Failed", d.ns, "!=", d.fs)
+		}
+		if d.ns > 0 && nsToFloatString(-d.ns) != "-"+d.fs {
+			t.Error("Failed on negative", d.ns, "!=", d.fs)
+		}
+	}
+}
+
+func TestFloatStringToNs(t *testing.T) {
+	for _, d := range []struct {
+		ns int64
+		fs string
+	}{
+		{0, "0"},
+		{0, "0."},
+		{0, "0.0"},
+		{0, "0.0000000001"},
+		{1, "0.000000001"},
+		{1000, "0.000001"},
+		{1000000, "0.001"},
+		{100000000, "0.1"},
+		{100000000, "0.10"},
+		{100000000, "0.1000000001"},
+		{1000000000, "1"},
+		{1000000000, "1."},
+		{1000000000, "1.0"},
+		{10000000000, "10"},
+		{12345678912, "12.345678912"},
+		{12345678912, "12.3456789129"},
+		{12345678912, "12.34567891299"},
+		{12345678910, "12.34567891"},
+		{12345678900, "12.3456789"},
+		{12345678000, "12.345678"},
+		{12345670000, "12.34567"},
+		{12345600000, "12.3456"},
+		{12345000000, "12.345"},
+		{12340000000, "12.34"},
+		{12300000000, "12.3"},
+		{12000000000, "12"},
+		{10000000000, "10"},
+		// This is a typical value which has more bits in than a float64
+		{1347717491123123123, "1347717491.123123123"},
+	} {
+		ns, err := floatStringToNs(d.fs)
+		if err != nil {
+			t.Error("Failed conversion", err)
+		}
+		if ns != d.ns {
+			t.Error("Failed", d.fs, "!=", d.ns, "was", ns)
+		}
+		if d.ns > 0 {
+			ns, err := floatStringToNs("-" + d.fs)
+			if err != nil {
+				t.Error("Failed conversion", err)
+			}
+			if ns != -d.ns {
+				t.Error("Failed on negative", -d.ns, "!=", "-"+d.fs)
+			}
+		}
+	}
+
+	// These are expected to produce errors
+	for _, fs := range []string{
+		"",
+		".0",
+		" 1",
+		"- 1",
+		"- 1",
+		"1.-1",
+		"1.0.0",
+		"1x0",
+	} {
+		ns, err := floatStringToNs(fs)
+		if err == nil {
+			t.Error("Didn't produce expected error", fs, ns)
+		}
+	}
+
+}