diff --git a/backend/onedrive/api/types.go b/backend/onedrive/api/types.go index 1f644b034..6491f26bb 100644 --- a/backend/onedrive/api/types.go +++ b/backend/onedrive/api/types.go @@ -2,7 +2,10 @@ package api -import "time" +import ( + "strings" + "time" +) const ( timeFormat = `"` + time.RFC3339 + `"` @@ -93,6 +96,22 @@ type ItemReference struct { Path string `json:"path"` // Path that used to navigate to the item. Read/Write. } +// RemoteItemFacet groups data needed to reference a OneDrive remote item +type RemoteItemFacet struct { + ID string `json:"id"` // The unique identifier of the item within the remote Drive. Read-only. + Name string `json:"name"` // The name of the item (filename and extension). Read-write. + CreatedBy IdentitySet `json:"createdBy"` // Identity of the user, device, and application which created the item. Read-only. + LastModifiedBy IdentitySet `json:"lastModifiedBy"` // Identity of the user, device, and application which last modified the item. Read-only. + CreatedDateTime Timestamp `json:"createdDateTime"` // Date and time of item creation. Read-only. + LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // Date and time the item was last modified. Read-only. + Folder *FolderFacet `json:"folder"` // Folder metadata, if the item is a folder. Read-only. + File *FileFacet `json:"file"` // File metadata, if the item is a file. Read-only. + FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write. + ParentReference *ItemReference `json:"parentReference"` // Parent information, if the item has a parent. Read-write. + Size int64 `json:"size"` // Size of the item in bytes. Read-only. + WebURL string `json:"webUrl"` // URL that displays the resource in the browser. Read-only. +} + // FolderFacet groups folder-related data on OneDrive into a single structure type FolderFacet struct { ChildCount int64 `json:"childCount"` // Number of children contained immediately within this container. @@ -143,6 +162,7 @@ type Item struct { Description string `json:"description"` // Provide a user-visible description of the item. Read-write. Folder *FolderFacet `json:"folder"` // Folder metadata, if the item is a folder. Read-only. File *FileFacet `json:"file"` // File metadata, if the item is a file. Read-only. + RemoteItem *RemoteItemFacet `json:"remoteItem"` // Remote Item metadata, if the item is a remote shared item. Read-only. FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write. // Image *ImageFacet `json:"image"` // Image metadata, if the item is an image. Read-only. // Photo *PhotoFacet `json:"photo"` // Photo metadata, if the item is a photo. Read-only. @@ -228,3 +248,112 @@ type AsyncOperationStatus struct { PercentageComplete float64 `json:"percentageComplete"` // An float value between 0 and 100 that indicates the percentage complete. Status string `json:"status"` // A string value that maps to an enumeration of possible values about the status of the job. "notStarted | inProgress | completed | updating | failed | deletePending | deleteFailed | waiting" } + +// GetID returns a normalized ID of the item +// If DriveID is known it will be prefixed to the ID with # seperator +func (i *Item) GetID() string { + if i.IsRemote() && i.RemoteItem.ID != "" { + return i.RemoteItem.ParentReference.DriveID + "#" + i.RemoteItem.ID + } else if i.ParentReference != nil && strings.Index(i.ID, "#") == -1 { + return i.ParentReference.DriveID + "#" + i.ID + } + return i.ID +} + +// GetDriveID returns a normalized ParentReferance of the item +func (i *Item) GetDriveID() string { + return i.GetParentReferance().DriveID +} + +// GetName returns a normalized Name of the item +func (i *Item) GetName() string { + if i.IsRemote() && i.RemoteItem.Name != "" { + return i.RemoteItem.Name + } + return i.Name +} + +// GetFolder returns a normalized Folder of the item +func (i *Item) GetFolder() *FolderFacet { + if i.IsRemote() && i.RemoteItem.Folder != nil { + return i.RemoteItem.Folder + } + return i.Folder +} + +// GetFile returns a normalized File of the item +func (i *Item) GetFile() *FileFacet { + if i.IsRemote() && i.RemoteItem.File != nil { + return i.RemoteItem.File + } + return i.File +} + +// GetFileSystemInfo returns a normalized FileSystemInfo of the item +func (i *Item) GetFileSystemInfo() *FileSystemInfoFacet { + if i.IsRemote() && i.RemoteItem.FileSystemInfo != nil { + return i.RemoteItem.FileSystemInfo + } + return i.FileSystemInfo +} + +// GetSize returns a normalized Size of the item +func (i *Item) GetSize() int64 { + if i.IsRemote() && i.RemoteItem.Size != 0 { + return i.RemoteItem.Size + } + return i.Size +} + +// GetWebURL returns a normalized WebURL of the item +func (i *Item) GetWebURL() string { + if i.IsRemote() && i.RemoteItem.WebURL != "" { + return i.RemoteItem.WebURL + } + return i.WebURL +} + +// GetCreatedBy returns a normalized CreatedBy of the item +func (i *Item) GetCreatedBy() IdentitySet { + if i.IsRemote() && i.RemoteItem.CreatedBy != (IdentitySet{}) { + return i.RemoteItem.CreatedBy + } + return i.CreatedBy +} + +// GetLastModifiedBy returns a normalized LastModifiedBy of the item +func (i *Item) GetLastModifiedBy() IdentitySet { + if i.IsRemote() && i.RemoteItem.LastModifiedBy != (IdentitySet{}) { + return i.RemoteItem.LastModifiedBy + } + return i.LastModifiedBy +} + +// GetCreatedDateTime returns a normalized CreatedDateTime of the item +func (i *Item) GetCreatedDateTime() Timestamp { + if i.IsRemote() && i.RemoteItem.CreatedDateTime != (Timestamp{}) { + return i.RemoteItem.CreatedDateTime + } + return i.CreatedDateTime +} + +// GetLastModifiedDateTime returns a normalized LastModifiedDateTime of the item +func (i *Item) GetLastModifiedDateTime() Timestamp { + if i.IsRemote() && i.RemoteItem.LastModifiedDateTime != (Timestamp{}) { + return i.RemoteItem.LastModifiedDateTime + } + return i.LastModifiedDateTime +} + +// GetParentReferance returns a normalized ParentReferance of the item +func (i *Item) GetParentReferance() *ItemReference { + if i.IsRemote() && i.ParentReference == nil { + return i.RemoteItem.ParentReference + } + return i.ParentReference +} + +// IsRemote checks if item is a remote item +func (i *Item) IsRemote() bool { + return i.RemoteItem != nil +} diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index e9e34e165..4450c205b 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -75,6 +75,7 @@ var ( oauthBusinessResource = oauth2.SetAuthURLParam("resource", discoveryServiceURL) chunkSize = fs.SizeSuffix(10 * 1024 * 1024) + sharedURL = "https://api.onedrive.com/v1.0/drives" // root URL for remote shared resources ) // Register with Fs @@ -325,6 +326,7 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Respon resp, err = f.srv.CallJSON(&opts, nil, &info) return shouldRetry(resp, err) }) + return info, resp, err } @@ -357,6 +359,7 @@ func NewFs(name, root string) (fs.Fs, error) { // business account setup oauthConfig = oauthBusinessConfig rootURL = resourceURL + "_api/v2.0/drives/me" + sharedURL = resourceURL + "_api/v2.0/drives" // update the URL in the AuthOptions oauthBusinessResource = oauth2.SetAuthURLParam("resource", resourceURL) @@ -482,21 +485,18 @@ func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err er } return "", false, err } - if info.Folder == nil { + if info.GetFolder() == nil { return "", false, errors.New("found file when looking for folder") } - return info.ID, true, nil + return info.GetID(), true, nil } // CreateDir makes a directory with pathID as parent and name leaf -func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) { - // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) +func (f *Fs) CreateDir(dirID, leaf string) (newID string, err error) { + // fs.Debugf(f, "CreateDir(%q, %q)\n", dirID, leaf) var resp *http.Response var info *api.Item - opts := rest.Opts{ - Method: "POST", - Path: "/items/" + pathID + "/children", - } + opts := newOptsCall(dirID, "POST", "/children") mkdir := api.CreateItemRequest{ Name: replaceReservedChars(leaf), ConflictBehavior: "fail", @@ -509,8 +509,9 @@ func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) { //fmt.Printf("...Error %v\n", err) return "", err } + //fmt.Printf("...Id %q\n", *info.Id) - return info.ID, nil + return info.GetID(), nil } // list the objects into the function supplied @@ -527,10 +528,8 @@ type listAllFn func(*api.Item) bool func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { // Top parameter asks for bigger pages of data // https://dev.onedrive.com/odata/optional-query-parameters.htm - opts := rest.Opts{ - Method: "GET", - Path: "/items/" + dirID + "/children?top=1000", - } + opts := newOptsCall(dirID, "GET", "/children?top=1000") + OUTER: for { var result api.ListChildrenResponse @@ -547,7 +546,7 @@ OUTER: } for i := range result.Value { item := &result.Value[i] - isFolder := item.Folder != nil + isFolder := item.GetFolder() != nil if isFolder { if filesOnly { continue @@ -560,7 +559,7 @@ OUTER: if item.Deleted != nil { continue } - item.Name = restoreReservedChars(item.Name) + item.Name = restoreReservedChars(item.GetName()) if fn(item) { found = true break OUTER @@ -595,13 +594,15 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { } var iErr error _, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { - remote := path.Join(dir, info.Name) - if info.Folder != nil { + remote := path.Join(dir, info.GetName()) + folder := info.GetFolder() + if folder != nil { // cache the directory ID for later lookups - f.dirCache.Put(remote, info.ID) - d := fs.NewDir(remote, time.Time(info.LastModifiedDateTime)).SetID(info.ID) - if info.Folder != nil { - d.SetItems(info.Folder.ChildCount) + id := info.GetID() + f.dirCache.Put(remote, id) + d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id) + if folder != nil { + d.SetItems(folder.ChildCount) } entries = append(entries, d) } else { @@ -674,11 +675,9 @@ func (f *Fs) Mkdir(dir string) error { // deleteObject removes an object by ID func (f *Fs) deleteObject(id string) error { - opts := rest.Opts{ - Method: "DELETE", - Path: "/items/" + id, - NoResponse: true, - } + opts := newOptsCall(id, "DELETE", "") + opts.NoResponse = true + return f.pacer.Call(func() (bool, error) { resp, err := f.srv.Call(&opts) return shouldRetry(resp, err) @@ -814,17 +813,17 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { } // Copy the object - opts := rest.Opts{ - Method: "POST", - Path: "/items/" + srcObj.id + "/action.copy", - ExtraHeaders: map[string]string{"Prefer": "respond-async"}, - NoResponse: true, - } + opts := newOptsCall(srcObj.id, "POST", "/action.copy") + opts.ExtraHeaders = map[string]string{"Prefer": "respond-async"} + opts.NoResponse = true + + id, _, _ := parseDirID(directoryID) + replacedLeaf := replaceReservedChars(leaf) copy := api.CopyItemRequest{ Name: &replacedLeaf, ParentReference: api.ItemReference{ - ID: directoryID, + ID: id, }, } var resp *http.Response @@ -891,14 +890,14 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { } // Move the object - opts := rest.Opts{ - Method: "PATCH", - Path: "/items/" + srcObj.id, - } + opts := newOptsCall(srcObj.id, "PATCH", "") + + id, _, _ := parseDirID(directoryID) + move := api.MoveItemRequest{ Name: replaceReservedChars(leaf), ParentReference: &api.ItemReference{ - ID: directoryID, + ID: id, }, // We set the mod time too as it gets reset otherwise FileSystemInfo: &api.FileSystemInfoFacet{ @@ -1013,35 +1012,37 @@ func (o *Object) Size() int64 { // setMetaData sets the metadata from info func (o *Object) setMetaData(info *api.Item) (err error) { - if info.Folder != nil { + if info.GetFolder() != nil { return errors.Wrapf(fs.ErrorNotAFile, "%q", o.remote) } o.hasMetaData = true - o.size = info.Size + o.size = info.GetSize() // Docs: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes // // We use SHA1 for onedrive personal and QuickXorHash for onedrive for business - if info.File != nil { - o.mimeType = info.File.MimeType - if info.File.Hashes.Sha1Hash != "" { - o.sha1 = strings.ToLower(info.File.Hashes.Sha1Hash) + file := info.GetFile() + if file != nil { + o.mimeType = file.MimeType + if file.Hashes.Sha1Hash != "" { + o.sha1 = strings.ToLower(file.Hashes.Sha1Hash) } - if info.File.Hashes.QuickXorHash != "" { - h, err := base64.StdEncoding.DecodeString(info.File.Hashes.QuickXorHash) + if file.Hashes.QuickXorHash != "" { + h, err := base64.StdEncoding.DecodeString(file.Hashes.QuickXorHash) if err != nil { - fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", info.File.Hashes.QuickXorHash, err) + fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", file.Hashes.QuickXorHash, err) } else { o.quickxorhash = hex.EncodeToString(h) } } } - if info.FileSystemInfo != nil { - o.modTime = time.Time(info.FileSystemInfo.LastModifiedDateTime) + fileSystemInfo := info.GetFileSystemInfo() + if fileSystemInfo != nil { + o.modTime = time.Time(fileSystemInfo.LastModifiedDateTime) } else { - o.modTime = time.Time(info.LastModifiedDateTime) + o.modTime = time.Time(info.GetLastModifiedDateTime()) } - o.id = info.ID + o.id = info.GetID() return nil } @@ -1080,9 +1081,20 @@ func (o *Object) ModTime() time.Time { // setModTime sets the modification time of the local fs object func (o *Object) setModTime(modTime time.Time) (*api.Item, error) { - opts := rest.Opts{ - Method: "PATCH", - Path: "/root:/" + rest.URLPathEscape(o.srvPath()), + var opts rest.Opts + _, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false) + _, drive, rootURL := parseDirID(directoryID) + if drive != "" { + opts = rest.Opts{ + Method: "PATCH", + RootURL: rootURL, + Path: "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()), + } + } else { + opts = rest.Opts{ + Method: "PATCH", + Path: "/root:/" + rest.URLPathEscape(o.srvPath()), + } } update := api.SetFileSystemInfo{ FileSystemInfo: api.FileSystemInfoFacet{ @@ -1119,11 +1131,9 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { } fs.FixRangeOption(options, o.size) var resp *http.Response - opts := rest.Opts{ - Method: "GET", - Path: "/items/" + o.id + "/content", - Options: options, - } + opts := newOptsCall(o.id, "GET", "/content") + opts.Options = options + err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(&opts) return shouldRetry(resp, err) @@ -1141,9 +1151,20 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { // createUploadSession creates an upload session for the object func (o *Object) createUploadSession(modTime time.Time) (response *api.CreateUploadResponse, err error) { - opts := rest.Opts{ - Method: "POST", - Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/upload.createSession", + leaf, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false) + id, drive, rootURL := parseDirID(directoryID) + var opts rest.Opts + if drive != "" { + opts = rest.Opts{ + Method: "POST", + RootURL: rootURL, + Path: "/" + drive + "/items/" + id + ":/" + rest.URLPathEscape(leaf) + ":/upload.createSession", + } + } else { + opts = rest.Opts{ + Method: "POST", + Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/upload.createSession", + } } createRequest := api.CreateUploadRequest{} createRequest.Item.FileSystemInfo.CreatedDateTime = api.Timestamp(modTime) @@ -1251,11 +1272,24 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (i // uploadSinglepart uploads a file as a single part func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) { var resp *http.Response - opts := rest.Opts{ - Method: "PUT", - Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content", - ContentLength: &size, - Body: in, + var opts rest.Opts + _, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false) + _, drive, rootURL := parseDirID(directoryID) + if drive != "" { + opts = rest.Opts{ + Method: "PUT", + RootURL: rootURL, + Path: "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content", + ContentLength: &size, + Body: in, + } + } else { + opts = rest.Opts{ + Method: "PUT", + Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content", + ContentLength: &size, + Body: in, + } } // for go1.8 (see release notes) we must nil the Body if we want a // "Content-Length: 0" header which onedrive requires for all files. @@ -1269,6 +1303,7 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) ( if err != nil { return nil, err } + err = o.setMetaData(info) if err != nil { return nil, err @@ -1315,6 +1350,30 @@ func (o *Object) ID() string { return o.id } +func newOptsCall(id string, method string, route string) (opts rest.Opts) { + id, drive, rootURL := parseDirID(id) + + if drive != "" { + return rest.Opts{ + Method: method, + RootURL: rootURL, + Path: "/" + drive + "/items/" + id + route, + } + } + return rest.Opts{ + Method: method, + Path: "/items/" + id + route, + } +} + +func parseDirID(ID string) (string, string, string) { + if strings.Index(ID, "#") >= 0 { + s := strings.Split(ID, "#") + return s[1], s[0], sharedURL + } + return ID, "", "" +} + // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil)