pax_global_header 0000666 0000000 0000000 00000000064 14417540156 0014521 g ustar 00root root 0000000 0000000 52 comment=7db1a0188b1add9d88898dba4abf9968bd25f745
gocbcore-10.2.3/ 0000775 0000000 0000000 00000000000 14417540156 0013367 5 ustar 00root root 0000000 0000000 gocbcore-10.2.3/.golangci.yml 0000664 0000000 0000000 00000000730 14417540156 0015753 0 ustar 00root root 0000000 0000000 run:
tests: false
skip-files:
- logging.go # Logging has some utility functions that are useful to have around which get flagged up
linters:
enable:
- bodyclose
- revive
- gosec
- unconvert
linters-settings:
revive:
set-exit-status: true
min-confidence: 0.81
rules:
- name: var-naming
arguments: [["HTTP", "ASCII", "IP", "TTL", "URL", "TLS", "JSON"]]
errcheck:
check-type-assertions: true
check-blank: true
gocbcore-10.2.3/LICENSE 0000664 0000000 0000000 00000026136 14417540156 0014404 0 ustar 00root root 0000000 0000000
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
gocbcore-10.2.3/Makefile 0000664 0000000 0000000 00000001207 14417540156 0015027 0 ustar 00root root 0000000 0000000 devsetup:
go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.39.0
go get github.com/vektra/mockery/.../
test:
go test ./...
fasttest:
go test -short ./...
cover:
go test -coverprofile=cover.out ./...
lint:
golangci-lint run -v
check: lint
go test -cover -race ./...
bench:
go test -run ^$$ -bench . --disable-logger
updatemocks:
mockery -name dispatcher -output . -testonly -inpkg
mockery -name configManager -output . -testonly -inpkg
mockery -name httpComponentInterface -output . -testonly -inpkg
.PHONY: all test devsetup fasttest lint cover checkerrs checkfmt checkvet checkiea checkspell check bench updatemocks
gocbcore-10.2.3/README.md 0000664 0000000 0000000 00000001376 14417540156 0014655 0 ustar 00root root 0000000 0000000 # Couchbase Go Core
This package provides the underlying Couchbase IO for the gocb project.
If you are looking for the Couchbase Go SDK, you are probably looking for
[gocb](https://github.com/couchbase/gocb).
## Branching Strategy
The gocbcore library maintains a branch for each previous major revision
of its API. These branches are introduced just prior to any API breaking
changes. Active work is performed on the master branch, with releases
being performed as tags. Work made on master which are not yet part of a
tagged released should be considered liable to change.
## License
Copyright 2017 Couchbase Inc.
Licensed under the Apache License, Version 2.0.
See
[LICENSE](https://github.com/couchbase/gocbcore/blob/master/LICENSE)
for further details.
gocbcore-10.2.3/RELEASE_NOTES.md 0000664 0000000 0000000 00000045417 14417540156 0015754 0 ustar 00root root 0000000 0000000 # Release Notes
## Version 10.2.3 (18 April 2023)
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1401](GOCBC-1401):
Exposed SeqNo on DCP rollback error.
* [https://issues.couchbase.com/browse/GOCBC-1403](GOCBC-1403):
Fixed issue where cccp poller would wait for a cluster config before starting.
## Version 10.2.2 (22 March 2023)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1393](GOCBC-1393):
Altered the behaviour of retries for enhanced prepared statements.
* [https://issues.couchbase.com/browse/GOCBC-1395](GOCBC-1395):
Improved timeout errors on http based services.
## Version 10.2.1 (22 February 2023)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1362](GOCBC-1362):
Added support for sending unsupported frames with `memd.Conn`.
* [https://issues.couchbase.com/browse/GOCBC-1322](GOCBC-1322):
Added volatile stability support for kv range scan.
Added volatile stability support for waiting for a config snapshot to be available.
* [https://issues.couchbase.com/browse/GOCBC-1373](GOCBC-1373):
Added support for query error code 1197.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1376](GOCBC-1376):
Fixed issue where lost cleanup would log an incorrectly formatted log line.
* [https://issues.couchbase.com/browse/GOCBC-1387](GOCBC-1387):
Fixed issue where an edge case could trigger a race between releasing connection buffers and reading on the connection - leading to a panic.
* [https://issues.couchbase.com/browse/GOCBC-1388](GOCBC-1388):
Fixed issue where the SDK could not connect to all nodes when `NoTLSSeedNode` is set in environments where multiple nodes are identifying as 127.0.0.1 (and so do not set a hostname in the cluster config).
## Version 10.2.0 (19 October 2022)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1159](GOCBC-1159):
Added support for refreshing the DNS SRV record when cluster becomes uncontactable.
* [https://issues.couchbase.com/browse/GOCBC-1284](GOCBC-1284):
Significant refactoring work to kv bootstrap.
* [https://issues.couchbase.com/browse/GOCBC-1303](GOCBC-1303):
Added `ServerWaitBackoff` to agent options.
* [https://issues.couchbase.com/browse/GOCBC-1316](GOCBC-1316):
Added support for transactions ExtInsertExisting.
* [https://issues.couchbase.com/browse/GOCBC-1328](GOCBC-1328):
Only fallback from cccp polling to http polling once all nodes tried.
* [https://issues.couchbase.com/browse/GOCBC-1331](GOCBC-1331):
Added support for pipelining fetching a config into kv bootstrap.
* [https://issues.couchbase.com/browse/GOCBC-1335](GOCBC-1335):
Updated logging to include address and pointer location in memdclient.
* [https://issues.couchbase.com/browse/GOCBC-1351](GOCBC-1351):
Updated error message logged on auth failures.
* [https://issues.couchbase.com/browse/GOCBC-1352](GOCBC-1352):
Added support for trusting the system cert store when TLS enabled and no cert provider registered.
* [https://issues.couchbase.com/browse/GOCBC-1356](GOCBC-1356):
Updated the behaviour when `MutateIn` or `Add` returns `NOT_STORED` to return a `ErrDocumentExists`.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1347](GOCBC-1347):
Fixed issue where a nil agent value could cause logging `TransactionATRLocation` to log a panic.
* [https://issues.couchbase.com/browse/GOCBC-1348](GOCBC-1348):
Fixed issue where a race on creating a client record could lead to a panic.
## Version 10.1.5 (21 September 2022)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1293](GOCBC-1293):
Added support for resource units.
* [https://issues.couchbase.com/browse/GOCBC-1332](GOCBC-1332):
Added deadlines to collections operations options.
* [https://issues.couchbase.com/browse/GOCBC-1339](GOCBC-1339):
Removed support for `CleanupWatchATRs` from `TransactionsConfig`.
Note that whilst this field still exists it is *not* used internally, it is included only for API level backward compatibility.
* [https://issues.couchbase.com/browse/GOCBC-1340](GOCBC-1340):
Added support for automatically starting lost cleanup on `TransactionsConfig` `CustomATRLocation`.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1338](GOCBC-1338):
Fixed issue where `lazyCircuitBreaker` was not using 64-bit aligned values.
### Known Issues
* [https://issues.couchbase.com/browse/GOCBC-1347](GOCBC-1347):
Known issue where a nil agent value could cause logging `TransactionATRLocation` to log a panic.
* [https://issues.couchbase.com/browse/GOCBC-1348](GOCBC-1348):
Known issue where a race on creating a client record can lead to a panic.
## Version 10.1.4 (20 July 2022)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1246](GOCBC-1246):
Added support for `TransactionLogger` to `TransactionOptions`.
* [https://issues.couchbase.com/browse/GOCBC-1314](GOCBC-1314):
Improved logging in the lost transactions process.
* [https://issues.couchbase.com/browse/GOCBC-1318](GOCBC-1318):
Changed `WaitUntilReady` to always wait for any explicitly defined services to be online.
* [https://issues.couchbase.com/browse/GOCBC-1319](GOCBC-1319):
Added a `String` implemented to `memd.Packet`.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1320](GOCBC-1320):
Fixed issue where vbucket hashing function wasn't masking out the 16th bit of the key.
## Version 10.1.3 (22 June 2022)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1264](GOCBC-1264):
Added more documentation to `AgentConfig`.
* [https://issues.couchbase.com/browse/GOCBC-1298](GOCBC-1298):
* [https://issues.couchbase.com/browse/GOCBC-1299](GOCBC-1299):
Masked the underlying cause of `TransactionOperationFailedError`.
* [https://issues.couchbase.com/browse/GOCBC-1159](GOCBC-1159):
Made improvements to handle a rebalance during a freeze in serverless environments.
* [https://issues.couchbase.com/browse/GOCBC-1283](GOCBC-1283):
Update forward compatibility errors to include document details.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1300](GOCBC-1300):
Added collection unknown check to `ProcessATR` to improve lost cleanup deleted collection handling.
* [https://issues.couchbase.com/browse/GOCBC-1304](GOCBC-1304):
Fixed issue where lost cleanup would block the SDK response thread for a connection.
* [https://issues.couchbase.com/browse/GOCBC-1301](GOCBC-1301):
Fixed issue where `addLostCleanupLocation` was left nil after `ResumeTransactionAttempt` called.
## Version 10.1.2 (26 April 2022)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1265](GOCBC-1265):
Bundle Capella CA certificates with the SDK.
## Version 10.1.1 (15 March 2022)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1221](GOCBC-1221):
Added support for improved query error handling.
* [https://issues.couchbase.com/browse/GOCBC-1238](GOCBC-1238):
Add config option to set the connection read buffer size.
* [https://issues.couchbase.com/browse/GOCBC-1242](GOCBC-1242):
Drain DCP queue on non-user initiated EOF.
* [https://issues.couchbase.com/browse/GOCBC-1221](GOCBC-1244):
Updated dependencies.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1248](GOCBC-1248):
Fixed issue where a hard close of a memdclient during a graceful close could trigger a panic.
* [https://issues.couchbase.com/browse/GOCBC-1256](GOCBC-1256):
Fixed issue where config polling would fallback to using the http poller, when no http addresses are registered for use.
* [https://issues.couchbase.com/browse/GOCBC-1258](GOCBC-1258):
Fixed issue where log redaction tags were not closed correctly.
## Version 10.1.0 (15 February 2022)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/TXNG-127](TXNG-127):
Integrate transactions into SDK.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1232](GOCBC-1232):
Fixed issue where DCP stream End could race with request cancellation (due to rebalance, etc...).
* [https://issues.couchbase.com/browse/GOCBC-1233](GOCBC-1233):
Fixed issue where Agent close could hang if called whilst auth request in flight.
## Version 10.0.7 (24 January 2022)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1216](GOCBC-1216):
Add support for missing memcached status code 0x8d
* [https://issues.couchbase.com/browse/GOCBC-1222](GOCBC-1222):
Updated memcached connections to use a `sync.Pool` for buffers for readers, to help reduce memory footprint.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1214](GOCBC-1214):
Fixed issue where nodes "actual" IP could be used for internal config instead of seed address when `NoTLSSeedNode` in use.
## Version 10.0.6 (14 December 2021)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1190](GOCBC-1190):
Added internal stability support for sending queries to specific nodes.
* [https://issues.couchbase.com/browse/GOCBC-1196](GOCBC-1196):
* Added error body and status code to analytics, query, search, view errors.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1205](GOCBC-1205):
Fixed issue where tracer spans were not always being finished.
* [https://issues.couchbase.com/browse/GOCBC-1206](GOCBC-1206):
Fixed issue where metrics were always incorrectly reporting very short durations for operations.
* [https://issues.couchbase.com/browse/GOCBC-1208](GOCBC-1208):
Fixed issue where cluster config polling would fallback to HTTP polling even when there was no bucket.
* [https://issues.couchbase.com/browse/GOCBC-1209](GOCBC-1209):
Fixed issue where the ns server connection string scheme wouldn't work for DCP.
## Version 10.0.5 (16 November 2021)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1179](GOCBC-1179):
Gracefully close memdclients on pipeline shutdown/reconnect.
* [https://issues.couchbase.com/browse/GOCBC-1180](GOCBC-1180):
Added support for the ns_server connection string scheme and seed (i.e. localhost) poller.
* [https://issues.couchbase.com/browse/GOCBC-1181](GOCBC-1181):
Added support for `ReconfigureSecurity` function.
* [https://issues.couchbase.com/browse/GOCBC-1182](GOCBC-1182):
Request error map v2 from the server.
* [https://issues.couchbase.com/browse/GOCBC-1193](GOCBC-1193):
Added the response body to query errors.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1194](GOCBC-1194):
Fixed issue where we wouldn't try to build a route config with all seed nodes for default network type before trying external network type.
## Version 10.0.4 (19 October 2021)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1178](GOCBC-1178):
Don't remove poller controller watcher from cluster config updates.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1177](GOCBC-1177):
Fixed issue where a connection being closed by the server during bootstrap could cause the SDK to loop reconnect without backoff.
## Version 10.0.3 (21 September 2021)
###New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1162](GOCBC-1162):
Added support for initially bootstrapping the SDK over nonTLS when TLS is in use.
* [https://issues.couchbase.com/browse/GOCBC-1169](GOCBC-1169):
Updated query streamer so that additional calls to `NextRow` return nil rather than panic.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1160](GOCBC-1160):
Fixed issue where HTTP header used for user impersonation was incorrect.
* [https://issues.couchbase.com/browse/GOCBC-1163](GOCBC-1163):
Fixed issue where cluster config parsing would check existence of wrong ports for TLS (although then assign correct ports).
## Version 10.0.2 (17 August 2021)
###New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1146](GOCBC-1146):
Added support for user impersonation to non-KV services.
* [https://issues.couchbase.com/browse/GOCBC-1148](GOCBC-1148):
Added support for forcibly reconnecting all connections.
* [https://issues.couchbase.com/browse/GOCBC-1150](GOCBC-1150):
Update user impersonation options for KV to use a string rather than []byte.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1139](GOCBC-1139):
Fixed issue where DCP agent would try to use SCRAM auth with TLS enabled, causing LDAP usage to always fail bootstrap.
* [https://issues.couchbase.com/browse/GOCBC-1147](GOCBC-1147):
Fixed issue where failing to fetch the error map during bootstrap would lead to bootstrap hanging.
## Version 10.0.1 (15 July 2021)
### Fixed Issues
* Fixed issue where modules file contained incorrect gocbcore version.
## Version 10.0.0 (15 July 2021) (Do not use, see v10.0.1)
###New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-901](GOCBC-901):
Broke the `AgentConfig` up into grouped components.
* [https://issues.couchbase.com/browse/GOCBC-1008](GOCBC-1008):
Updated mutate in to return cas mismatch error rather than document exists when doing a replace.
* [https://issues.couchbase.com/browse/GOCBC-1062](GOCBC-1062):
Added support for DCP snapshot marker v2 and v2.1.
* [https://issues.couchbase.com/browse/GOCBC-1081](GOCBC-1081):
During CCCP polling don't retry request if the error is request cancelled.
* [https://issues.couchbase.com/browse/GOCBC-1130](GOCBC-1130):
Updated Query error handling to return an authentication error on error code 13104.
* [https://issues.couchbase.com/browse/GOCBC-1087](GOCBC-1087):
Added support for communicating with Eventing and Backup services.
* [https://issues.couchbase.com/browse/GOCBC-1093](GOCBC-1093):
Added support for `RevEpoch` in bucket configs.
* [https://issues.couchbase.com/browse/GOCBC-1044](GOCBC-1044):
* [https://issues.couchbase.com/browse/GOCBC-1128](GOCBC-1128):
Added `Meter` interface and operation level response latency metric.
* [https://issues.couchbase.com/browse/GOCBC-1133](GOCBC-1133):
Remove `ViewQuery` from `AgentGroup`.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1135](GOCBC-1135):
Fixed issue where cmd traces could be ended twice in some scenarios when operation was cancelled.
## Version 9.1.5 (15 June 2021)
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1095](GOCBC-1095):
Fixed issue where SDK was parsing view error contents incorrectly.
* [https://issues.couchbase.com/browse/GOCBC-1102](GOCBC-1102):
Fixed issue where `WaitUntilReady` wouldn't recover if one of the HTTP based services returned an error.
* [https://issues.couchbase.com/browse/GOCBC-1106](GOCBC-1106):
* [https://issues.couchbase.com/browse/GOCBC-1112](GOCBC-1112):
Fixed issues where fts responses were being parsed incorrectly.
* [https://issues.couchbase.com/browse/GOCBC-1127](GOCBC-1127):
Fixed issue where query errors could be parsed incorrectly.
## Version 9.1.4 (20 April 2021)
###New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1071](GOCBC-1071):
Updated SDK to use new protocol level changes for get collection id.
* [https://issues.couchbase.com/browse/GOCBC-1068](GOCBC-1068):
Dropped log level to warn for when applying a cluster config object is preempted.
* [https://issues.couchbase.com/browse/GOCBC-1079](GOCBC-1079):
During bootstrap don't retry authentication if the error is request cancelled.
* [https://issues.couchbase.com/browse/GOCBC-1081](GOCBC-1081):
During CCCP polling don't retry request if the error is request cancelled.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1080](GOCBC-1080):
Fixed issue where SDK would always rebuild connections on first cluster config fetched against server 7.0.
* [https://issues.couchbase.com/browse/GOCBC-1082](GOCBC-1082):
Fixed issue where bootstrapping a node during an SDK wide reconnect would cause a delay in connecting to that node.
* [https://issues.couchbase.com/browse/GOCBC-1088](GOCBC-1088):
Fixed issue where the poller controller could deadlock if a node reported a bucket not found at the same time as CCCP successfully fetched a cluster config for the first time.
## Version 9.1.3 (16 March 2021)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1056](GOCBC-1056):
Various performance improvements to reduce CPU level.
* [https://issues.couchbase.com/browse/GOCBC-1068](GOCBC-1068):
Dropped the log level for preempted config updates.
* [https://issues.couchbase.com/browse/GOCBC-940](GOCBC-940):
Updated the tracing interfaces and orphaned response logging output.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1066](GOCBC-1066):
Fixed issue which could cause the config pollers to panic.
## Version 9.1.2 (16 February 2021)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1041](GOCBC-1041):
Dropped the log level for memdclient read failures to warn, from error.
* [https://issues.couchbase.com/browse/GOCBC-1046](GOCBC-1046):
Added `MaxTTl` to `ManifestCollection`.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1042](GOCBC-1042):
Fixed issue where bucket names were not being correctly escaped.
* [https://issues.couchbase.com/browse/GOCBC-1050](GOCBC-1050):
Fixed issue where the diagnostics component could panic if an operation was cancelled by the user after it had already been internally cancelled.
## Version 9.1.1 (19 January 2021)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-1032](GOCBC-1032):
Added support for bucket capability support verification to agent, at API stability internal.
* [https://issues.couchbase.com/browse/GOCBC-1030](GOCBC-1030):
Added support for internal cancellation of bootstrap before completion, allowing pipeline clients to shutdown without waiting for bootstrap to complete (such as on connection takeover).
Added support to fallback to http config fetching if select bucket fails with a valid fallback error, allowing for faster config fetching against non-kv nodes.
## Version 9.1.0 (15 December 2020)
### New Features and Behavioral Changes
* [https://issues.couchbase.com/browse/GOCBC-854](GOCBC-854):
Added support for user impersonation.
* [https://issues.couchbase.com/browse/GOCBC-1013](GOCBC-1013):
Added support for `StatsKeys` and `StatsChunks` to `SingleServerStats` to support responses for stats keys such as `connections` which contain complex objects per packet.
### Fixed Issues
* [https://issues.couchbase.com/browse/GOCBC-1016](GOCBC-1016):
Fixed issue where creating an agent with no bucket and a non-default port HTTP address could lead to a panic in `WaitForReady`.
(Note: `WaitForReady` will *never* return success in this scenario)
* [https://issues.couchbase.com/browse/GOCBC-1028](GOCBC-1028):
Fixed issue where bootstrapping against a non-kv node could never successfully fully connect.
gocbcore-10.2.3/agent.go 0000664 0000000 0000000 00000064776 14417540156 0015040 0 ustar 00root root 0000000 0000000 // Package gocbcore implements methods for low-level communication
// with a Couchbase Server cluster.
package gocbcore
import (
"crypto/x509"
"errors"
"fmt"
"net"
"net/http"
"strings"
"sync"
"time"
)
// Agent represents the base client handling connections to a Couchbase Server.
// This is used internally by the higher level classes for communicating with the cluster,
// it can also be used to perform more advanced operations with a cluster.
type Agent struct {
clientID string
bucketName string
defaultRetryStrategy RetryStrategy
pollerController configPollerController
kvMux *kvMux
httpMux *httpMux
dialer *memdClientDialerComponent
cfgManager *configManagementComponent
errMap *errMapComponent
collections *collectionsComponent
tracer *tracerComponent
http *httpComponent
diagnostics *diagnosticsComponent
crud *crudComponent
observe *observeComponent
stats *statsComponent
n1ql *n1qlQueryComponent
analytics *analyticsQueryComponent
search *searchQueryComponent
views *viewQueryComponent
zombieLogger *zombieLoggerComponent
// These connection settings are only ever changed when ForceReconnect or ReconfigureSecurity are called.
connectionSettingsLock sync.Mutex
auth AuthProvider
authMechanisms []AuthMechanism
tlsConfig *dynTLSConfig
srvDetails *srvDetails
shutdownSig chan struct{}
}
// HTTPClient returns a pre-configured HTTP Client for communicating with
// Couchbase Server. You must still specify authentication information
// for any dispatched requests.
func (agent *Agent) HTTPClient() *http.Client {
return agent.http.cli
}
type srvDetails struct {
Addrs routeEndpoints
Record SRVRecord
}
// CreateAgent creates an agent for performing normal operations.
func CreateAgent(config *AgentConfig) (*Agent, error) {
return createAgent(config)
}
func createAgent(config *AgentConfig) (*Agent, error) {
logInfof("SDK Version: gocbcore/%s", goCbCoreVersionStr)
logInfof("Creating new agent: %+v", config)
tracer := config.TracerConfig.Tracer
if tracer == nil {
tracer = noopTracer{}
}
tracerCmpt := newTracerComponent(tracer, config.BucketName, config.TracerConfig.NoRootTraceSpans, config.MeterConfig.Meter)
c := &Agent{
clientID: formatCbUID(randomCbUID()),
bucketName: config.BucketName,
tracer: tracerCmpt,
defaultRetryStrategy: config.DefaultRetryStrategy,
errMap: newErrMapManager(config.BucketName),
auth: config.SecurityConfig.Auth,
shutdownSig: make(chan struct{}),
}
tlsConfig, err := setupTLSConfig(config.SeedConfig.MemdAddrs, config.SecurityConfig)
if err != nil {
return nil, err
}
c.tlsConfig = tlsConfig
httpIdleConnTimeout := 4500 * time.Millisecond
if config.HTTPConfig.IdleConnectionTimeout > 0 {
httpIdleConnTimeout = config.HTTPConfig.IdleConnectionTimeout
}
httpConnectTimeout := 30 * time.Second
if config.HTTPConfig.ConnectTimeout > 0 {
httpConnectTimeout = config.HTTPConfig.ConnectTimeout
}
circuitBreakerConfig := config.CircuitBreakerConfig
userAgent := config.UserAgent
useMutationTokens := config.IoConfig.UseMutationTokens
disableDecompression := config.CompressionConfig.DisableDecompression
useCompression := config.CompressionConfig.Enabled
useCollections := config.IoConfig.UseCollections
useJSONHello := !config.IoConfig.DisableJSONHello
usePITRHello := config.IoConfig.EnablePITRHello
useXErrorHello := !config.IoConfig.DisableXErrorHello
useSyncReplicationHello := !config.IoConfig.DisableSyncReplicationHello
useResourceUnits := config.InternalConfig.EnableResourceUnitsTrackingHello
compressionMinSize := 32
compressionMinRatio := 0.83
useDurations := config.IoConfig.UseDurations
useOutOfOrder := config.IoConfig.UseOutOfOrderResponses
kvConnectTimeout := 7000 * time.Millisecond
if config.KVConfig.ConnectTimeout > 0 {
kvConnectTimeout = config.KVConfig.ConnectTimeout
}
serverWaitTimeout := 5 * time.Second
if config.KVConfig.ServerWaitBackoff > 0 {
serverWaitTimeout = config.KVConfig.ServerWaitBackoff
}
kvPoolSize := 1
if config.KVConfig.PoolSize > 0 {
kvPoolSize = config.KVConfig.PoolSize
}
maxQueueSize := 2048
if config.KVConfig.MaxQueueSize > 0 {
maxQueueSize = config.KVConfig.MaxQueueSize
}
kvBufferSize := uint(0)
if config.KVConfig.ConnectionBufferSize > 0 {
kvBufferSize = config.KVConfig.ConnectionBufferSize
}
confHTTPRetryDelay := 10 * time.Second
if config.ConfigPollerConfig.HTTPRetryDelay > 0 {
confHTTPRetryDelay = config.ConfigPollerConfig.HTTPRetryDelay
}
confHTTPRedialPeriod := 10 * time.Second
if config.ConfigPollerConfig.HTTPRedialPeriod > 0 {
confHTTPRedialPeriod = config.ConfigPollerConfig.HTTPRedialPeriod
}
confHTTPMaxWait := 5 * time.Second
if config.ConfigPollerConfig.HTTPMaxWait > 0 {
confHTTPMaxWait = config.ConfigPollerConfig.HTTPMaxWait
}
confCccpMaxWait := 3 * time.Second
if config.ConfigPollerConfig.CccpMaxWait > 0 {
confCccpMaxWait = config.ConfigPollerConfig.CccpMaxWait
}
confCccpPollPeriod := 2500 * time.Millisecond
if config.ConfigPollerConfig.CccpPollPeriod > 0 {
confCccpPollPeriod = config.ConfigPollerConfig.CccpPollPeriod
}
if config.CompressionConfig.MinSize > 0 {
compressionMinSize = config.CompressionConfig.MinSize
}
if config.CompressionConfig.MinRatio > 0 {
compressionMinRatio = config.CompressionConfig.MinRatio
if compressionMinRatio >= 1.0 {
compressionMinRatio = 1.0
}
}
if c.defaultRetryStrategy == nil {
c.defaultRetryStrategy = newFailFastRetryStrategy()
}
c.authMechanisms = authMechanismsFromConfig(config.SecurityConfig.AuthMechanisms, tlsConfig != nil)
httpEpList := routeEndpoints{}
var srcHTTPAddrs []routeEndpoint
for _, hostPort := range config.SeedConfig.HTTPAddrs {
if config.SecurityConfig.UseTLS && !config.SecurityConfig.NoTLSSeedNode {
ep := routeEndpoint{
Address: fmt.Sprintf("https://%s", hostPort),
IsSeedNode: true,
}
httpEpList.SSLEndpoints = append(httpEpList.SSLEndpoints, ep)
srcHTTPAddrs = append(srcHTTPAddrs, ep)
} else {
ep := routeEndpoint{
Address: fmt.Sprintf("http://%s", hostPort),
IsSeedNode: true,
}
httpEpList.NonSSLEndpoints = append(httpEpList.NonSSLEndpoints, ep)
srcHTTPAddrs = append(srcHTTPAddrs, ep)
}
}
if config.OrphanReporterConfig.Enabled {
zombieLoggerInterval := 10 * time.Second
zombieLoggerSampleSize := 10
if config.OrphanReporterConfig.ReportInterval > 0 {
zombieLoggerInterval = config.OrphanReporterConfig.ReportInterval
}
if config.OrphanReporterConfig.SampleSize > 0 {
zombieLoggerSampleSize = config.OrphanReporterConfig.SampleSize
}
c.zombieLogger = newZombieLoggerComponent(zombieLoggerInterval, zombieLoggerSampleSize)
go c.zombieLogger.Start()
}
kvServerList := routeEndpoints{}
var srcMemdAddrs []routeEndpoint
for _, seed := range config.SeedConfig.MemdAddrs {
if config.SecurityConfig.UseTLS && !config.SecurityConfig.NoTLSSeedNode {
kvServerList.SSLEndpoints = append(kvServerList.SSLEndpoints, routeEndpoint{
Address: seed,
IsSeedNode: true,
})
srcMemdAddrs = kvServerList.SSLEndpoints
} else {
kvServerList.NonSSLEndpoints = append(kvServerList.NonSSLEndpoints, routeEndpoint{
Address: seed,
IsSeedNode: true,
})
srcMemdAddrs = kvServerList.NonSSLEndpoints
}
}
if config.SeedConfig.SRVRecord != nil {
c.srvDetails = &srvDetails{
Addrs: kvServerList,
Record: *config.SeedConfig.SRVRecord,
}
}
c.cfgManager = newConfigManager(
configManagerProperties{
NetworkType: config.IoConfig.NetworkType,
SrcMemdAddrs: srcMemdAddrs,
SrcHTTPAddrs: srcHTTPAddrs,
UseTLS: tlsConfig != nil,
NoTLSSeedNode: config.SecurityConfig.NoTLSSeedNode,
},
)
c.dialer = newMemdClientDialerComponent(
memdClientDialerProps{
ServerWaitTimeout: serverWaitTimeout,
KVConnectTimeout: kvConnectTimeout,
ClientID: c.clientID,
CompressionMinSize: compressionMinSize,
CompressionMinRatio: compressionMinRatio,
DisableDecompression: disableDecompression,
NoTLSSeedNode: config.SecurityConfig.NoTLSSeedNode,
ConnBufSize: kvBufferSize,
},
bootstrapProps{
HelloProps: helloProps{
CollectionsEnabled: useCollections,
MutationTokensEnabled: useMutationTokens,
CompressionEnabled: useCompression,
DurationsEnabled: useDurations,
OutOfOrderEnabled: useOutOfOrder,
JSONFeatureEnabled: useJSONHello,
XErrorFeatureEnabled: useXErrorHello,
SyncReplicationEnabled: useSyncReplicationHello,
PITRFeatureEnabled: usePITRHello,
ResourceUnitsEnabled: useResourceUnits,
},
Bucket: c.bucketName,
UserAgent: userAgent,
ErrMapManager: c.errMap,
},
circuitBreakerConfig,
c.zombieLogger,
c.tracer,
c.cfgManager,
)
c.kvMux = newKVMux(
kvMuxProps{
QueueSize: maxQueueSize,
PoolSize: kvPoolSize,
CollectionsEnabled: useCollections,
NoTLSSeedNode: config.SecurityConfig.NoTLSSeedNode,
},
c.cfgManager,
c.errMap,
c.tracer,
c.dialer,
&kvMuxState{
tlsConfig: tlsConfig,
authMechanisms: c.authMechanisms,
auth: config.SecurityConfig.Auth,
},
)
c.collections = newCollectionIDManager(
collectionIDProps{
MaxQueueSize: config.KVConfig.MaxQueueSize,
DefaultRetryStrategy: c.defaultRetryStrategy,
},
c.kvMux,
c.tracer,
c.cfgManager,
)
c.httpMux = newHTTPMux(
circuitBreakerConfig,
c.cfgManager,
&httpClientMux{tlsConfig: tlsConfig, auth: config.SecurityConfig.Auth},
config.SecurityConfig.NoTLSSeedNode,
)
c.http = newHTTPComponent(
httpComponentProps{
UserAgent: userAgent,
DefaultRetryStrategy: c.defaultRetryStrategy,
},
httpClientProps{
maxIdleConns: config.HTTPConfig.MaxIdleConns,
maxIdleConnsPerHost: config.HTTPConfig.MaxIdleConnsPerHost,
idleTimeout: httpIdleConnTimeout,
connectTimeout: httpConnectTimeout,
},
c.httpMux,
c.tracer,
)
if len(config.SeedConfig.MemdAddrs) == 0 && config.BucketName == "" {
// The http poller can't run without a bucket. We don't trigger an error for this case
// because AgentGroup users who use memcached buckets on non-default ports will end up here.
logDebugf("No bucket name specified and only http addresses specified, not running config poller")
c.diagnostics = newDiagnosticsComponent(c.kvMux, c.httpMux, c.http, c.bucketName, c.defaultRetryStrategy, nil)
} else {
var poller configPollerController
if config.SecurityConfig.NoTLSSeedNode {
poller = newSeedConfigController(srcHTTPAddrs[0].Address, c.bucketName,
httpPollerProperties{
httpComponent: c.http,
confHTTPRetryDelay: confHTTPRetryDelay,
confHTTPRedialPeriod: confHTTPRedialPeriod,
confHTTPMaxWait: confHTTPMaxWait,
}, c.cfgManager)
} else {
var httpPoller *httpConfigController
if c.bucketName != "" {
httpPoller = newHTTPConfigController(
c.bucketName,
httpPollerProperties{
httpComponent: c.http,
confHTTPRetryDelay: confHTTPRetryDelay,
confHTTPRedialPeriod: confHTTPRedialPeriod,
confHTTPMaxWait: confHTTPMaxWait,
},
c.httpMux,
c.cfgManager,
)
}
poller = newPollerController(
newCCCPConfigController(
cccpPollerProperties{
confCccpMaxWait: confCccpMaxWait,
confCccpPollPeriod: confCccpPollPeriod,
},
c.kvMux,
c.cfgManager,
c.isPollingFallbackError,
c.onCCCPNoConfigFromAnyNode,
),
httpPoller,
c.cfgManager,
c.isPollingFallbackError,
)
}
c.pollerController = poller
c.diagnostics = newDiagnosticsComponent(c.kvMux, c.httpMux, c.http, c.bucketName, c.defaultRetryStrategy, c.pollerController)
}
c.dialer.AddBootstrapFailHandler(c.diagnostics)
c.dialer.AddCCCPUnsupportedHandler(c)
c.cfgManager.AddConfigWatcher(c.dialer)
c.observe = newObserveComponent(c.collections, c.defaultRetryStrategy, c.tracer, c.kvMux)
c.crud = newCRUDComponent(c.collections, c.defaultRetryStrategy, c.tracer, c.errMap, c.kvMux, disableDecompression)
c.stats = newStatsComponent(c.kvMux, c.defaultRetryStrategy, c.tracer)
c.n1ql = newN1QLQueryComponent(c.http, c.cfgManager, c.tracer)
c.analytics = newAnalyticsQueryComponent(c.http, c.tracer)
c.search = newSearchQueryComponent(c.http, c.tracer)
c.views = newViewQueryComponent(c.http, c.tracer)
// Kick everything off.
cfg := &routeConfig{
kvServerList: kvServerList,
mgmtEpList: httpEpList,
revID: -1,
}
c.httpMux.OnNewRouteConfig(cfg)
c.kvMux.OnNewRouteConfig(cfg)
if c.pollerController != nil {
go c.pollerController.Run()
}
return c, nil
}
// Close shuts down the agent, disconnecting from all servers and failing
// any outstanding operations with ErrShutdown.
func (agent *Agent) Close() error {
poller := agent.pollerController
if poller != nil {
poller.Stop()
}
routeCloseErr := agent.kvMux.Close()
if agent.zombieLogger != nil {
agent.zombieLogger.Stop()
}
if poller != nil {
// Wait for our external looper goroutines to finish, note that if the
// specific looper wasn't used, it will be a nil value otherwise it
// will be an open channel till its closed to signal completion.
pollerCh := poller.Done()
if pollerCh != nil {
<-pollerCh
}
}
// Close the transports so that they don't hold open goroutines.
agent.http.Close()
close(agent.shutdownSig)
return routeCloseErr
}
// ClientID returns the unique id for this agent
func (agent *Agent) ClientID() string {
return agent.clientID
}
// MemdEps returns all the available endpoints for performing KV/DCP operations (using the memcached binary protocol).
// As apposed to other endpoints, these will have the 'couchbase(s)://' scheme prefix.
func (agent *Agent) MemdEps() []string {
snapshot, err := agent.kvMux.PipelineSnapshot()
if err != nil {
return []string{}
}
return snapshot.state.KVEps()
}
// CapiEps returns all the available endpoints for performing
// map-reduce queries.
func (agent *Agent) CapiEps() []string {
return agent.httpMux.CapiEps()
}
// MgmtEps returns all the available endpoints for performing
// management queries.
func (agent *Agent) MgmtEps() []string {
return agent.httpMux.MgmtEps()
}
// N1qlEps returns all the available endpoints for performing
// N1QL queries.
func (agent *Agent) N1qlEps() []string {
return agent.httpMux.N1qlEps()
}
// FtsEps returns all the available endpoints for performing
// FTS queries.
func (agent *Agent) FtsEps() []string {
return agent.httpMux.FtsEps()
}
// CbasEps returns all the available endpoints for performing
// CBAS queries.
func (agent *Agent) CbasEps() []string {
return agent.httpMux.CbasEps()
}
// EventingEps returns all the available endpoints for managing/interacting with the Eventing Service.
func (agent *Agent) EventingEps() []string {
return agent.httpMux.EventingEps()
}
// GSIEps returns all the available endpoints for managing/interacting with the GSI Service.
func (agent *Agent) GSIEps() []string {
return agent.httpMux.GSIEps()
}
// BackupEps returns all the available endpoints for managing/interacting with the Backup Service.
func (agent *Agent) BackupEps() []string {
return agent.httpMux.BackupEps()
}
// HasCollectionsSupport verifies whether or not collections are available on the agent.
func (agent *Agent) HasCollectionsSupport() bool {
return agent.kvMux.SupportsCollections()
}
// IsSecure returns whether this client is connected via SSL.
func (agent *Agent) IsSecure() bool {
return agent.kvMux.IsSecure()
}
// UsingGCCCP returns whether or not the Agent is currently using GCCCP polling.
func (agent *Agent) UsingGCCCP() bool {
return agent.kvMux.SupportsGCCCP()
}
// HasSeenConfig returns whether or not the Agent has seen a valid cluster config. This does not mean that the agent
// currently has active connections.
// Volatile: This API is subject to change at any time.
func (agent *Agent) HasSeenConfig() (bool, error) {
seen, err := agent.kvMux.ConfigRev()
if err != nil {
return false, err
}
return seen > -1, nil
}
// WaitUntilReady is used to verify that the SDK has been able to establish connections to the cluster.
// If no strategy is set then a fast fail retry strategy will be applied - only RetryReason that are set to always
// retry will be retried. This includes for WaitUntilReady, that is the SDK will wait until connections succeed
// or report a connection error - as soon as a connection error is reported WaitUntilReady will fail and return that
// error.
// Connection time errors are also be subject to KvConfig.ServerWaitBackoff. This is the period of time that the SDK
// will wait before attempting to reconnect to a node.
func (agent *Agent) WaitUntilReady(deadline time.Time, opts WaitUntilReadyOptions, cb WaitUntilReadyCallback) (PendingOp, error) {
forceWait := true
if len(opts.ServiceTypes) == 0 {
forceWait = false
opts.ServiceTypes = []ServiceType{MemdService}
}
return agent.diagnostics.WaitUntilReady(deadline, forceWait, opts, cb)
}
// ConfigSnapshot returns a snapshot of the underlying configuration currently in use.
func (agent *Agent) ConfigSnapshot() (*ConfigSnapshot, error) {
return agent.kvMux.ConfigSnapshot()
}
// WaitForConfigSnapshot returns a snapshot of the underlying configuration currently in use, once one is available.
// Volatile: This API is subject to change at any time.
func (agent *Agent) WaitForConfigSnapshot(deadline time.Time, opts WaitForConfigSnapshotOptions, cb WaitForConfigSnapshotCallback) (PendingOp, error) {
return agent.kvMux.WaitForConfigSnapshot(deadline, cb)
}
// BucketName returns the name of the bucket that the agent is using, if any.
// Uncommitted: This API may change in the future.
func (agent *Agent) BucketName() string {
return agent.bucketName
}
// ForceReconnect gracefully rebuilds all connections being used by the agent.
// Any persistent in flight requests (e.g. DCP) will be terminated with ErrForcedReconnect.
//
// Internal: This should never be used and is not supported.
func (agent *Agent) ForceReconnect() {
agent.connectionSettingsLock.Lock()
auth := agent.auth
mechs := agent.authMechanisms
tlsConfig := agent.tlsConfig
agent.connectionSettingsLock.Unlock()
agent.kvMux.ForceReconnect(tlsConfig, mechs, auth, true)
}
// ReconfigureSecurityOptions are the options available to the ReconfigureSecurity function.
type ReconfigureSecurityOptions struct {
UseTLS bool
// If is nil will default to the TLSRootCAProvider already in use by the agent.
TLSRootCAProvider func() *x509.CertPool
Auth AuthProvider
// AuthMechanisms is the list of mechanisms that the SDK can use to attempt authentication.
// Note that if you add PLAIN to the list, this will cause credential leakage on the network
// since PLAIN sends the credentials in cleartext. It is disabled by default to prevent downgrade attacks. We
// recommend using a TLS connection if using PLAIN.
// If is nil will default to the AuthMechanisms already in use by the Agent.
AuthMechanisms []AuthMechanism
}
// ReconfigureSecurity updates the security configuration being used by the agent. This includes the ability to
// toggle TLS on and off.
//
// Calling this function will cause all underlying connections to be reconnected. The exception to this is the
// connection to the seed node (usually localhost), which will only be reconnected if the AuthProvider is provided
// on the options.
//
// This function can only be called when the seed poller is in use i.e. when the ns_server scheme is used.
// Internal: This should never be used and is not supported.
func (agent *Agent) ReconfigureSecurity(opts ReconfigureSecurityOptions) error {
_, ok := agent.pollerController.(*seedConfigController)
if !ok {
return errors.New("reconfigure tls is only supported when the agent is in ns server mode")
}
var authProvided bool
auth := opts.Auth
mechs := opts.AuthMechanisms
agent.connectionSettingsLock.Lock()
if auth == nil {
auth = agent.auth
} else {
authProvided = true
}
if len(mechs) == 0 {
mechs = agent.authMechanisms
}
var tlsConfig *dynTLSConfig
if opts.UseTLS {
if opts.TLSRootCAProvider == nil {
return wrapError(errInvalidArgument, "must provide TLSRootCAProvider when UseTLS is true")
}
tlsConfig = createTLSConfig(auth, opts.TLSRootCAProvider)
}
agent.auth = auth
agent.authMechanisms = mechs
agent.tlsConfig = tlsConfig
agent.connectionSettingsLock.Unlock()
agent.cfgManager.UseTLS(tlsConfig != nil)
agent.kvMux.ForceReconnect(tlsConfig, mechs, auth, authProvided)
agent.httpMux.UpdateTLS(tlsConfig, auth)
return nil
}
func (agent *Agent) onCCCPUnsupported(err error) {
// If this error is a legitimate fallback reason then we should immediately start the http poller.
// This should always be a poller fallback error but lets just be sure.
if agent.pollerController != nil && agent.isPollingFallbackError(err) {
agent.pollerController.ForceHTTPPoller()
}
}
func (agent *Agent) isPollingFallbackError(err error) bool {
return isPollingFallbackError(err, agent.bucketName)
}
type srvAgent interface {
srv() *srvDetails
setSRVAddrs(routeEndpoints)
routeConfigWatchers() []routeConfigWatcher
resetConfig()
IsSecure() bool
stopped() <-chan struct{}
}
func (agent *Agent) srv() *srvDetails {
return agent.srvDetails
}
func (agent *Agent) setSRVAddrs(addrs routeEndpoints) {
agent.srvDetails.Addrs = addrs
}
func (agent *Agent) routeConfigWatchers() []routeConfigWatcher {
return agent.cfgManager.Watchers()
}
func (agent *Agent) resetConfig() {
// Reset the config manager to accept the next config that the poller fetches.
// This is safe to do here, we're blocking the poller from fetching a config and if we're here then
// we can't be performing ops.
agent.cfgManager.ResetConfig()
// Reset the dialer so that the next connections to bootstrap fetch a config and kick off the poller again.
agent.dialer.ResetConfig()
}
func (agent *Agent) onCCCPNoConfigFromAnyNode(err error) {
onCCCPNoConfigFromAnyNode(agent, err)
}
func (agent *Agent) stopped() <-chan struct{} {
return agent.shutdownSig
}
// The CCCP poller suddenly becoming unable to fetch a config from any node in the cluster is the trigger
// for checking if we need to try refresh the DNS SRV record that we used to initially connect.
// Note that we don't need locking around of this because there is only one poller active at any given time
// and we're blocking it here.
func onCCCPNoConfigFromAnyNode(agent srvAgent, err error) {
srvDetails := agent.srv()
if srvDetails == nil {
return
}
// We only want to refresh the SRV record under certain circumstances, namely that we can't connect to the cluster.
var opErr *net.OpError
if !errors.As(err, &opErr) {
return
}
logInfof("Refreshing SRV record: %s", srvDetails.Record)
var addrs []*net.SRV
for {
_, addrs, err = net.LookupSRV(srvDetails.Record.Scheme, srvDetails.Record.Proto, srvDetails.Record.Host)
if err != nil {
if isLogRedactionLevelFull() {
logInfof("Failed to lookup SRV record: %s", redactSystemData(err))
} else {
logInfof("Failed to lookup SRV record: %s", err)
}
}
if len(addrs) > 0 {
break
}
select {
case <-agent.stopped():
return
case <-time.After(10 * time.Second):
}
}
// If any of the addresses in the SRV record match an address that we already know then we can say that the
// cluster has not moved and bail out.
useTLS := agent.IsSecure()
var memdAddrs []routeEndpoint
if useTLS {
memdAddrs = srvDetails.Addrs.SSLEndpoints
} else {
memdAddrs = srvDetails.Addrs.NonSSLEndpoints
}
logAddrs := append([]routeEndpoint(nil), memdAddrs...)
if isLogRedactionLevelFull() {
for i, addr := range logAddrs {
logAddrs[i].Address = redactSystemData(addr)
}
}
logInfof("Found new addrs for SRV record: %v", logAddrs)
for _, addr := range addrs {
host := fmt.Sprintf("%s:%d", strings.TrimSuffix(addr.Target, "."), addr.Port)
for _, seed := range memdAddrs {
if host == seed.Address {
logInfof("Found already known matching address, not refreshing system")
return
}
}
}
logInfof("No matching address known, refreshing system")
agent.resetConfig()
kvServerList := routeEndpoints{}
for _, seed := range addrs {
host := fmt.Sprintf("%s:%d", strings.TrimSuffix(seed.Target, "."), seed.Port)
if useTLS {
kvServerList.SSLEndpoints = append(kvServerList.SSLEndpoints, routeEndpoint{
Address: host,
IsSeedNode: true,
})
} else {
kvServerList.NonSSLEndpoints = append(kvServerList.NonSSLEndpoints, routeEndpoint{
Address: host,
IsSeedNode: true,
})
}
}
// Build a new fake config to kick off the pipelines again, this will make the kvmux stop the old pipelines
// and create new connections to the new addresses.
newCfg := &routeConfig{
kvServerList: kvServerList,
revID: -1,
}
watchers := agent.routeConfigWatchers()
for _, watcher := range watchers {
watcher.OnNewRouteConfig(newCfg)
}
// Update the addresses we hold so that if the SRV changes again then we can correctly check the new vs old
// addresses.
agent.setSRVAddrs(kvServerList)
}
func authMechanismsFromConfig(authMechanisms []AuthMechanism, useTLS bool) []AuthMechanism {
if len(authMechanisms) == 0 {
if useTLS {
authMechanisms = []AuthMechanism{PlainAuthMechanism}
} else {
// No user specified auth mechanisms so set our defaults.
authMechanisms = []AuthMechanism{
ScramSha512AuthMechanism,
ScramSha256AuthMechanism,
ScramSha1AuthMechanism}
}
} else if !useTLS {
// The user has specified their own mechanisms and not using TLS so we check if they've set PLAIN.
for _, mech := range authMechanisms {
if mech == PlainAuthMechanism {
logWarnf("PLAIN sends credentials in plaintext, this will cause credential leakage on the network")
}
}
}
return authMechanisms
}
func setupTLSConfig(addrs []string, config SecurityConfig) (*dynTLSConfig, error) {
var tlsConfig *dynTLSConfig
if config.UseTLS {
if config.TLSRootCAProvider == nil {
logDebugf("TLS enabled with no root ca provider - trusting system cert pool and Capella root CA")
pool, err := x509.SystemCertPool()
if err != nil {
return nil, wrapError(err, "failed to load system cert pool")
}
pool.AppendCertsFromPEM(capellaRootCA)
config.TLSRootCAProvider = func() *x509.CertPool {
return pool
}
}
tlsConfig = createTLSConfig(config.Auth, config.TLSRootCAProvider)
} else {
var endsInCloud bool
for _, host := range addrs {
if strings.HasSuffix(strings.Split(host, ":")[0], ".cloud.couchbase.com") {
endsInCloud = true
break
}
}
if endsInCloud {
logWarnf("TLS is required when connecting to Couchbase Capella. Please enable TLS by prefixing " +
"the connection string with \"couchbases://\" (note the final 's').")
}
}
return tlsConfig, nil
}
gocbcore-10.2.3/agent_config.go 0000664 0000000 0000000 00000047534 14417540156 0016356 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"crypto/x509"
"errors"
"fmt"
"io/ioutil"
"strconv"
"time"
"github.com/couchbase/gocbcore/v10/connstr"
)
func parseDurationOrInt(valStr string) (time.Duration, error) {
dur, err := time.ParseDuration(valStr)
if err != nil {
val, err := strconv.ParseInt(valStr, 10, 64)
if err != nil {
return 0, err
}
dur = time.Duration(val) * time.Millisecond
}
return dur, nil
}
// AgentConfig specifies the configuration options for creation of an Agent.
type AgentConfig struct {
BucketName string
UserAgent string
SeedConfig SeedConfig
SecurityConfig SecurityConfig
CompressionConfig CompressionConfig
ConfigPollerConfig ConfigPollerConfig
IoConfig IoConfig
KVConfig KVConfig
HTTPConfig HTTPConfig
DefaultRetryStrategy RetryStrategy
CircuitBreakerConfig CircuitBreakerConfig
OrphanReporterConfig OrphanReporterConfig
TracerConfig TracerConfig
MeterConfig MeterConfig
InternalConfig InternalConfig
}
// OrphanReporterConfig specifies options for controlling the orphan
// reporter which records when the SDK receives responses for requests
// that are no longer in the system (usually due to being timed out).
type OrphanReporterConfig struct {
Enabled bool
// ReportInterval is the time period used for how often a report is logged.
ReportInterval time.Duration
// SampleSize is the number of requests which will be reported.
SampleSize int
}
func (config OrphanReporterConfig) fromSpec(spec connstr.ResolvedConnSpec) (OrphanReporterConfig, error) {
if valStr, ok := fetchOption(spec, "orphaned_response_logging"); ok {
val, err := strconv.ParseBool(valStr)
if err != nil {
return OrphanReporterConfig{}, fmt.Errorf("orphaned_response_logging option must be a boolean")
}
config.Enabled = val
}
if valStr, ok := fetchOption(spec, "orphaned_response_logging_interval"); ok {
val, err := parseDurationOrInt(valStr)
if err != nil {
return OrphanReporterConfig{}, fmt.Errorf("orphaned_response_logging_interval option must be a number")
}
config.ReportInterval = val
}
if valStr, ok := fetchOption(spec, "orphaned_response_logging_sample_size"); ok {
val, err := strconv.ParseInt(valStr, 10, 64)
if err != nil {
return OrphanReporterConfig{}, fmt.Errorf("orphaned_response_logging_sample_size option must be a number")
}
config.SampleSize = int(val)
}
return config, nil
}
// SecurityConfig specifies options for controlling security related
// items such as TLS root certificates and verification skipping.
type SecurityConfig struct {
UseTLS bool
TLSRootCAProvider func() *x509.CertPool
// NoTLSSeedNode indicates that, even with UseTLS set to true, the SDK should always connect to the seed node
// over a non TLS connection. This means that the seed node should ALWAYS be localhost.
// This option must be used with the ConfigPollerConfig UseSeedPoller set to true.
// Internal: This should never be used and is not supported.
NoTLSSeedNode bool
Auth AuthProvider
// AuthMechanisms is the list of mechanisms that the SDK can use to attempt authentication.
// Note that if you add PLAIN to the list, this will cause credential leakage on the network
// since PLAIN sends the credentials in cleartext. It is disabled by default to prevent downgrade attacks. We
// recommend using a TLS connection if using PLAIN.
AuthMechanisms []AuthMechanism
}
func (config SecurityConfig) fromSpec(spec connstr.ResolvedConnSpec) (SecurityConfig, error) {
if spec.UseSsl {
cacertpaths := spec.Options["ca_cert_path"]
if len(cacertpaths) > 0 {
roots := x509.NewCertPool()
for _, path := range cacertpaths {
cacert, err := ioutil.ReadFile(path)
if err != nil {
return SecurityConfig{}, err
}
ok := roots.AppendCertsFromPEM(cacert)
if !ok {
return SecurityConfig{}, errInvalidCertificate
}
}
config.TLSRootCAProvider = func() *x509.CertPool {
return roots
}
}
config.UseTLS = true
}
if spec.NSServerHost != nil {
config.NoTLSSeedNode = true
}
return config, nil
}
// CompressionConfig specifies options for controlling compression applied to documents using KV.
type CompressionConfig struct {
Enabled bool
DisableDecompression bool
MinSize int
MinRatio float64
}
func (config CompressionConfig) fromSpec(spec connstr.ResolvedConnSpec) (CompressionConfig, error) {
if valStr, ok := fetchOption(spec, "compression"); ok {
val, err := strconv.ParseBool(valStr)
if err != nil {
return CompressionConfig{}, fmt.Errorf("compression option must be a boolean")
}
config.Enabled = val
}
if valStr, ok := fetchOption(spec, "compression_min_size"); ok {
val, err := strconv.ParseInt(valStr, 10, 64)
if err != nil {
return CompressionConfig{}, fmt.Errorf("compression_min_size option must be an int")
}
config.MinSize = int(val)
}
if valStr, ok := fetchOption(spec, "compression_min_ratio"); ok {
val, err := strconv.ParseFloat(valStr, 64)
if err != nil {
return CompressionConfig{}, fmt.Errorf("compression_min_size option must be an int")
}
config.MinRatio = val
}
return config, nil
}
// ConfigPollerConfig specifies options for controlling the cluster configuration pollers.
type ConfigPollerConfig struct {
HTTPRedialPeriod time.Duration
HTTPRetryDelay time.Duration
HTTPMaxWait time.Duration
CccpMaxWait time.Duration
CccpPollPeriod time.Duration
}
func (config ConfigPollerConfig) fromSpec(spec connstr.ResolvedConnSpec) (ConfigPollerConfig, error) {
if valStr, ok := fetchOption(spec, "config_poll_timeout"); ok {
val, err := parseDurationOrInt(valStr)
if err != nil {
return ConfigPollerConfig{}, fmt.Errorf("config poll timeout option must be a duration or a number")
}
config.CccpMaxWait = val
}
if valStr, ok := fetchOption(spec, "config_poll_interval"); ok {
val, err := parseDurationOrInt(valStr)
if err != nil {
return ConfigPollerConfig{}, fmt.Errorf("config pool interval option must be duration or a number")
}
config.CccpPollPeriod = val
}
// This option is experimental
if valStr, ok := fetchOption(spec, "http_redial_period"); ok {
val, err := parseDurationOrInt(valStr)
if err != nil {
return ConfigPollerConfig{}, fmt.Errorf("http redial period option must be a duration or a number")
}
config.HTTPRedialPeriod = val
}
// This option is experimental
if valStr, ok := fetchOption(spec, "http_retry_delay"); ok {
val, err := parseDurationOrInt(valStr)
if err != nil {
return ConfigPollerConfig{}, fmt.Errorf("http retry delay option must be a duration or a number")
}
config.HTTPRetryDelay = val
}
if valStr, ok := fetchOption(spec, "http_config_poll_timeout"); ok {
val, err := parseDurationOrInt(valStr)
if err != nil {
return ConfigPollerConfig{}, fmt.Errorf("http_config_poll_timeout option must be a duration or a number")
}
config.HTTPMaxWait = val
}
return config, nil
}
// IoConfig specifies IO related configuration options such as HELLO flags and the network type to use.
type IoConfig struct {
// NetworkType defines which network to use from the cluster config.
NetworkType string
UseMutationTokens bool
UseDurations bool
UseOutOfOrderResponses bool
DisableXErrorHello bool
DisableJSONHello bool
DisableSyncReplicationHello bool
EnablePITRHello bool
UseCollections bool
}
func (config IoConfig) fromSpec(spec connstr.ResolvedConnSpec) (IoConfig, error) {
if valStr, ok := fetchOption(spec, "network"); ok {
config.NetworkType = valStr
}
if valStr, ok := fetchOption(spec, "enable_mutation_tokens"); ok {
val, err := strconv.ParseBool(valStr)
if err != nil {
return IoConfig{}, fmt.Errorf("enable_mutation_tokens option must be a boolean")
}
config.UseMutationTokens = val
}
if valStr, ok := fetchOption(spec, "enable_server_durations"); ok {
val, err := strconv.ParseBool(valStr)
if err != nil {
return IoConfig{}, fmt.Errorf("server_duration option must be a boolean")
}
config.UseDurations = val
}
// This option is experimental
if valStr, ok := fetchOption(spec, "unordered_execution_enabled"); ok {
val, err := strconv.ParseBool(valStr)
if err != nil {
return IoConfig{}, fmt.Errorf("unordered_execution_enabled option must be a boolean")
}
config.UseOutOfOrderResponses = val
}
return config, nil
}
// TracerConfig specifies tracer related configuration options.
type TracerConfig struct {
Tracer RequestTracer
NoRootTraceSpans bool
}
// MeterConfig specifies meter related configuration options.
type MeterConfig struct {
Meter Meter
}
// HTTPConfig specifies http related configuration options.
type HTTPConfig struct {
// MaxIdleConns controls the maximum number of idle (keep-alive) connections across all hosts.
MaxIdleConns int
// MaxIdleConnsPerHost controls the maximum idle (keep-alive) connections to keep per-host.
MaxIdleConnsPerHost int
ConnectTimeout time.Duration
// IdleConnTimeout is the maximum amount of time an idle (keep-alive) connection will remain idle before closing
// itself.
IdleConnectionTimeout time.Duration
}
func (config HTTPConfig) fromSpec(spec connstr.ResolvedConnSpec) (HTTPConfig, error) {
if valStr, ok := fetchOption(spec, "max_idle_http_connections"); ok {
val, err := strconv.ParseInt(valStr, 10, 64)
if err != nil {
return HTTPConfig{}, fmt.Errorf("http max idle connections option must be a number")
}
config.MaxIdleConns = int(val)
}
if valStr, ok := fetchOption(spec, "max_perhost_idle_http_connections"); ok {
val, err := strconv.ParseInt(valStr, 10, 64)
if err != nil {
return HTTPConfig{}, fmt.Errorf("max_perhost_idle_http_connections option must be a number")
}
config.MaxIdleConnsPerHost = int(val)
}
if valStr, ok := fetchOption(spec, "idle_http_connection_timeout"); ok {
val, err := parseDurationOrInt(valStr)
if err != nil {
return HTTPConfig{}, fmt.Errorf("idle_http_connection_timeout option must be a duration or a number")
}
config.IdleConnectionTimeout = val
}
if valStr, ok := fetchOption(spec, "http_connect_timeout"); ok {
val, err := parseDurationOrInt(valStr)
if err != nil {
return HTTPConfig{}, fmt.Errorf("http_connect_timeout option must be a duration or a number")
}
config.ConnectTimeout = val
}
return config, nil
}
// KVConfig specifies kv related configuration options.
type KVConfig struct {
// ConnectTimeout is the timeout value to apply when dialling tcp connections.
ConnectTimeout time.Duration
// ServerWaitBackoff is the period of time that the SDK will wait before reattempting connection to a node after
// bootstrap fails against that node.
ServerWaitBackoff time.Duration
// The number of connections to create to each node.
PoolSize int
// The maximum number of requests that can be queued waiting to be sent to a node.
MaxQueueSize int
// Note: if you create multiple agents with different buffer sizes within the same environment then you will
// get indeterminate behaviour, the connections may not even use the provided buffer size.
ConnectionBufferSize uint
}
func (config KVConfig) fromSpec(spec connstr.ResolvedConnSpec) (KVConfig, error) {
if valStr, ok := fetchOption(spec, "kv_connect_timeout"); ok {
val, err := parseDurationOrInt(valStr)
if err != nil {
return KVConfig{}, fmt.Errorf("kv_connect_timeout option must be a duration or a number")
}
config.ConnectTimeout = val
}
// This option is experimental
if valStr, ok := fetchOption(spec, "kv_pool_size"); ok {
val, err := strconv.ParseInt(valStr, 10, 64)
if err != nil {
return KVConfig{}, fmt.Errorf("kv pool size option must be a number")
}
config.PoolSize = int(val)
}
// This option is experimental
if valStr, ok := fetchOption(spec, "max_queue_size"); ok {
val, err := strconv.ParseInt(valStr, 10, 64)
if err != nil {
return KVConfig{}, fmt.Errorf("max queue size option must be a number")
}
config.MaxQueueSize = int(val)
}
// This option is experimental
if valStr, ok := fetchOption(spec, "kv_buffer_size"); ok {
val, err := strconv.ParseInt(valStr, 10, 64)
if err != nil {
return KVConfig{}, fmt.Errorf("kv buffer size option must be a number")
}
config.ConnectionBufferSize = uint(val)
}
if valStr, ok := fetchOption(spec, "server_wait_backoff"); ok {
val, err := strconv.ParseInt(valStr, 10, 64)
if err != nil {
return KVConfig{}, fmt.Errorf("server wait backoff must be a number")
}
config.ServerWaitBackoff = time.Duration(val) * time.Millisecond
}
return config, nil
}
// SRVRecord describes the SRV record used to extract memd addresses in the SeedConfig.
type SRVRecord struct {
Proto string
Scheme string
Host string
}
func (srv SRVRecord) redacted() interface{} {
return fmt.Sprintf("%s %s %s", srv.Proto, srv.Scheme, redactSystemData(srv.Host))
}
// SeedConfig specifies initial seed configuration options such as addresses.
type SeedConfig struct {
HTTPAddrs []string
MemdAddrs []string
SRVRecord *SRVRecord
}
func (config SeedConfig) fromSpec(spec connstr.ResolvedConnSpec) (SeedConfig, error) {
// Grab the resolved hostnames into a set of string arrays
var httpHosts []string
for _, specHost := range spec.HttpHosts {
httpHosts = append(httpHosts, fmt.Sprintf("%s:%d", specHost.Host, specHost.Port))
}
var memdHosts []string
for _, specHost := range spec.MemdHosts {
memdHosts = append(memdHosts, fmt.Sprintf("%s:%d", specHost.Host, specHost.Port))
}
var nsServerHost string
if spec.NSServerHost != nil {
nsServerHost = fmt.Sprintf("%s:%d", spec.NSServerHost.Host, spec.NSServerHost.Port)
}
if nsServerHost != "" {
if len(httpHosts) > 0 || len(memdHosts) > 0 {
return SeedConfig{}, errors.New("ns_server host cannot be used alongside http or memd hosts")
}
httpHosts = append(httpHosts, nsServerHost)
}
// Get bootstrap_on option to determine which, if any, of the bootstrap nodes should be cleared
switch val, _ := fetchOption(spec, "bootstrap_on"); val {
case "http":
memdHosts = nil
if len(httpHosts) == 0 {
return SeedConfig{}, errors.New("bootstrap_on=http but no HTTP hosts in connection string")
}
case "cccp":
httpHosts = nil
if len(memdHosts) == 0 {
return SeedConfig{}, errors.New("bootstrap_on=cccp but no CCCP/Memcached hosts in connection string")
}
case "both":
if nsServerHost != "" {
return SeedConfig{}, errors.New("bootstrap_on=both but ns_server host in connection string")
}
case "":
// Do nothing
break
default:
// Don't advertise ns_server as an option
return SeedConfig{}, errors.New("bootstrap_on={http,cccp,both}")
}
config.MemdAddrs = memdHosts
config.HTTPAddrs = httpHosts
if spec.SrvRecord != nil {
config.SRVRecord = &SRVRecord{
Proto: spec.SrvRecord.Proto,
Scheme: spec.SrvRecord.Scheme,
Host: spec.SrvRecord.Host,
}
}
return config, nil
}
func (config SeedConfig) redacted() SeedConfig {
newConfig := SeedConfig{
HTTPAddrs: config.HTTPAddrs,
MemdAddrs: config.MemdAddrs,
SRVRecord: config.SRVRecord,
}
// The slices here are still pointing at config's underlying arrays
// so we need to make them not do that.
newConfig.HTTPAddrs = append([]string(nil), newConfig.HTTPAddrs...)
for i, addr := range newConfig.HTTPAddrs {
newConfig.HTTPAddrs[i] = redactSystemData(addr)
}
newConfig.MemdAddrs = append([]string(nil), newConfig.MemdAddrs...)
for i, addr := range newConfig.MemdAddrs {
newConfig.MemdAddrs[i] = redactSystemData(addr)
}
return newConfig
}
// InternalConfig specifies internal configs.
// Internal: This should never be used and is not supported.
type InternalConfig struct {
EnableResourceUnitsTrackingHello bool
}
func (config InternalConfig) fromSpec(spec connstr.ResolvedConnSpec) (InternalConfig, error) {
if valStr, ok := fetchOption(spec, "enable_resource_units"); ok {
val, err := strconv.ParseBool(valStr)
if err != nil {
return InternalConfig{}, fmt.Errorf("enable_resource_units option must be a boolean")
}
config.EnableResourceUnitsTrackingHello = val
}
return config, nil
}
func (config *AgentConfig) redacted() interface{} {
newConfig := *config
if isLogRedactionLevelFull() {
newConfig.SeedConfig = newConfig.SeedConfig.redacted()
if newConfig.BucketName != "" {
newConfig.BucketName = redactMetaData(newConfig.BucketName)
}
}
return newConfig
}
func fetchOption(spec connstr.ResolvedConnSpec, name string) (string, bool) {
optValue := spec.Options[name]
if len(optValue) == 0 {
return "", false
}
return optValue[len(optValue)-1], true
}
// FromConnStr populates the AgentConfig with information from a
// Couchbase Connection String.
// Supported options are:
//
// bootstrap_on (bool) - Specifies what protocol to bootstrap on (cccp, http).
// ca_cert_path (string) - Specifies the path to a CA certificate.
// network (string) - The network type to use.
// kv_connect_timeout (duration) - Maximum period to attempt to connect to cluster in ms.
// config_poll_interval (duration) - Period to wait between CCCP config polling in ms.
// config_poll_timeout (duration) - Maximum period of time to wait for a CCCP request.
// compression (bool) - Whether to enable network-wise compression of documents.
// compression_min_size (int) - The minimal size of the document in bytes to consider compression.
// compression_min_ratio (float64) - The minimal compress ratio (compressed / original) for the document to be sent compressed.
// enable_server_durations (bool) - Whether to enable fetching server operation durations.
// max_idle_http_connections (int) - Maximum number of idle http connections in the pool.
// max_perhost_idle_http_connections (int) - Maximum number of idle http connections in the pool per host.
// idle_http_connection_timeout (duration) - Maximum length of time for an idle connection to stay in the pool in ms.
// orphaned_response_logging (bool) - Whether to enable orphaned response logging.
// orphaned_response_logging_interval (duration) - How often to print the orphan log records.
// orphaned_response_logging_sample_size (int) - The maximum number of orphan log records to track.
// dcp_priority (int) - Specifies the priority to request from the Cluster when connecting for DCP.
// enable_dcp_expiry (bool) - Whether to enable the feature to distinguish between explicit delete and expired delete on DCP.
// http_redial_period (duration) - The maximum length of time for the HTTP poller to stay connected before reconnecting.
// http_retry_delay (duration) - The length of time to wait between HTTP poller retries if connecting fails.
// kv_pool_size (int) - The number of connections to create to each kv node.
// max_queue_size (int) - The maximum number of requests that can be queued for sending per connection.
// unordered_execution_enabled (bool) - Whether to enabled the "out of order responses" feature.
// server_wait_backoff (duration) -The period of time waited between kv reconnect attmepts to a node after connection failure
func (config *AgentConfig) FromConnStr(connStr string) error {
baseSpec, err := connstr.Parse(connStr)
if err != nil {
return err
}
spec, err := connstr.Resolve(baseSpec)
if err != nil {
return err
}
if spec.Bucket != "" {
config.BucketName = spec.Bucket
}
config.SeedConfig, err = config.SeedConfig.fromSpec(spec)
if err != nil {
return err
}
config.SecurityConfig, err = config.SecurityConfig.fromSpec(spec)
if err != nil {
return err
}
config.OrphanReporterConfig, err = config.OrphanReporterConfig.fromSpec(spec)
if err != nil {
return err
}
config.CompressionConfig, err = config.CompressionConfig.fromSpec(spec)
if err != nil {
return err
}
config.ConfigPollerConfig, err = config.ConfigPollerConfig.fromSpec(spec)
if err != nil {
return err
}
config.IoConfig, err = config.IoConfig.fromSpec(spec)
if err != nil {
return err
}
config.HTTPConfig, err = config.HTTPConfig.fromSpec(spec)
if err != nil {
return err
}
config.KVConfig, err = config.KVConfig.fromSpec(spec)
if err != nil {
return err
}
config.InternalConfig, err = config.InternalConfig.fromSpec(spec)
if err != nil {
return err
}
return nil
}
gocbcore-10.2.3/agent_config_test.go 0000664 0000000 0000000 00000054361 14417540156 0017411 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"testing"
"time"
)
func (suite *StandardTestSuite) TestAgentConfig_FromConnStr() {
connStr := "couchbase://10.112.192.101,10.112.192.102?bootstrap_on=cccp&network=external&kv_connect_timeout=100us"
config := &AgentConfig{}
err := config.FromConnStr(connStr)
if err != nil {
suite.T().Fatalf("Failed to execute FromConnStr: %v", err)
}
if config.KVConfig.ConnectTimeout != 100*time.Microsecond {
suite.T().Fatalf("Ex :%v", config.KVConfig.ConnectTimeout)
}
}
func (suite *StandardTestSuite) TestAgentConfig_Couchbase1() {
connStr := "couchbase://10.112.192.101"
config := &AgentConfig{}
err := config.FromConnStr(connStr)
if err != nil {
suite.T().Fatalf("Failed to execute FromConnStr: %v", err)
}
if len(config.SeedConfig.MemdAddrs) != 1 {
suite.T().Fatalf("Expected MemdAddrs to be len 1 but was %v", config.SeedConfig.MemdAddrs)
}
if len(config.SeedConfig.HTTPAddrs) != 1 {
suite.T().Fatalf("Expected MemdAddrs to be len 1 but was %v", config.SeedConfig.HTTPAddrs)
}
if config.SeedConfig.MemdAddrs[0] != "10.112.192.101:11210" {
suite.T().Fatalf("Expected address to be 10.112.192.101:11210 but was %v", config.SeedConfig.MemdAddrs[0])
}
if config.SeedConfig.HTTPAddrs[0] != "10.112.192.101:8091" {
suite.T().Fatalf("Expected address to be 10.112.192.101:8091 but was %v", config.SeedConfig.HTTPAddrs[0])
}
}
func (suite *StandardTestSuite) TestAgentConfig_Couchbase2() {
connStr := "couchbase://10.112.192.101,10.112.192.102"
config := &AgentConfig{}
err := config.FromConnStr(connStr)
if err != nil {
suite.T().Fatalf("Failed to execute FromConnStr: %v", err)
}
if len(config.SeedConfig.MemdAddrs) != 2 {
suite.T().Fatalf("Expected MemdAddrs to be len 2 but was %v", config.SeedConfig.MemdAddrs)
}
if len(config.SeedConfig.HTTPAddrs) != 2 {
suite.T().Fatalf("Expected MemdAddrs to be len 2 but was %v", config.SeedConfig.HTTPAddrs)
}
}
func (suite *StandardTestSuite) TestAgentConfig_DefaultHTTP() {
connStr := "http://10.112.192.101:8091"
config := &AgentConfig{}
err := config.FromConnStr(connStr)
if err != nil {
suite.T().Fatalf("Failed to execute FromConnStr: %v", err)
}
if len(config.SeedConfig.MemdAddrs) != 1 {
suite.T().Fatalf("Expected MemdAddrs to be len 1 but was %v", config.SeedConfig.MemdAddrs)
}
if len(config.SeedConfig.HTTPAddrs) != 1 {
suite.T().Fatalf("Expected MemdAddrs to be len 1 but was %v", config.SeedConfig.HTTPAddrs)
}
if config.SeedConfig.MemdAddrs[0] != "10.112.192.101:11210" {
suite.T().Fatalf("Expected address to be 10.112.192.101:11210 but was %v", config.SeedConfig.MemdAddrs[0])
}
if config.SeedConfig.HTTPAddrs[0] != "10.112.192.101:8091" {
suite.T().Fatalf("Expected address to be 10.112.192.101:8091 but was %v", config.SeedConfig.HTTPAddrs[0])
}
}
func (suite *StandardTestSuite) TestAgentConfig_NonDefaultHTTP() {
connStr := "http://10.112.192.101:9000"
config := &AgentConfig{}
err := config.FromConnStr(connStr)
if err != nil {
suite.T().Fatalf("Failed to execute FromConnStr: %v", err)
}
if len(config.SeedConfig.MemdAddrs) != 0 {
suite.T().Fatalf("Expected MemdAddrs to be len 0 but was %v", config.SeedConfig.MemdAddrs)
}
if len(config.SeedConfig.HTTPAddrs) != 1 {
suite.T().Fatalf("Expected MemdAddrs to be len 1 but was %v", config.SeedConfig.HTTPAddrs)
}
if config.SeedConfig.HTTPAddrs[0] != "10.112.192.101:9000" {
suite.T().Fatalf("Expected address to be 10.112.192.101:9000 but was %v", config.SeedConfig.HTTPAddrs[0])
}
}
func (suite *StandardTestSuite) TestAgentConfig_BootstrapOnCCCP() {
tests := []struct {
name string
connStr string
lenMemdAddrs int
lenHttpAddrs int
}{
{
name: "cccp",
connStr: "couchbase://10.112.192.101?bootstrap_on=cccp",
lenMemdAddrs: 1,
lenHttpAddrs: 0,
},
{
name: "http",
connStr: "couchbase://10.112.192.101?bootstrap_on=http",
lenMemdAddrs: 0,
lenHttpAddrs: 1,
},
{
name: "both",
connStr: "couchbase://10.112.192.101?bootstrap_on=both",
lenMemdAddrs: 1,
lenHttpAddrs: 1,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); err != nil {
t.Errorf("FromConnStr() error = %v", err)
}
if len(config.SeedConfig.MemdAddrs) != tt.lenMemdAddrs {
suite.T().Fatalf("Expected MemdAddrs to be len %d but was %v", tt.lenMemdAddrs, config.SeedConfig.HTTPAddrs)
}
if len(config.SeedConfig.HTTPAddrs) != tt.lenHttpAddrs {
suite.T().Fatalf("Expected HTTPAddrs to be len %d but was %v", tt.lenHttpAddrs, config.SeedConfig.HTTPAddrs)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_Network() {
tests := []struct {
name string
connStr string
expected string
}{
{
name: "external",
connStr: "couchbase://10.112.192.101?network=external",
expected: "external",
},
{
name: "default",
connStr: "couchbase://10.112.192.101?network=default",
expected: "default",
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); err != nil {
t.Errorf("FromConnStr() error = %v", err)
}
if config.IoConfig.NetworkType != tt.expected {
suite.T().Fatalf("Expected %s but was %s", tt.expected, config.IoConfig.NetworkType)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_KVConnectTimeout() {
tests := []struct {
name string
connStr string
expected time.Duration
wantErr bool
}{
{
name: "duration",
connStr: "couchbase://10.112.192.101?kv_connect_timeout=5000us",
expected: 5 * time.Millisecond,
},
{
name: "ms",
connStr: "couchbase://10.112.192.101?kv_connect_timeout=5",
expected: 5 * time.Millisecond,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?kv_connect_timeout=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if config.KVConfig.ConnectTimeout != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.KVConfig.ConnectTimeout)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_ConfigPollTimeout() {
tests := []struct {
name string
connStr string
expected time.Duration
wantErr bool
}{
{
name: "duration",
connStr: "couchbase://10.112.192.101?config_poll_timeout=5000us",
expected: 5 * time.Millisecond,
},
{
name: "ms",
connStr: "couchbase://10.112.192.101?config_poll_timeout=5",
expected: 5 * time.Millisecond,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?config_poll_timeout=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if config.ConfigPollerConfig.CccpMaxWait != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.ConfigPollerConfig.CccpMaxWait)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_ConfigPollPeriod() {
tests := []struct {
name string
connStr string
expected time.Duration
wantErr bool
}{
{
name: "duration",
connStr: "couchbase://10.112.192.101?config_poll_interval=5000us",
expected: 5 * time.Millisecond,
},
{
name: "ms",
connStr: "couchbase://10.112.192.101?config_poll_interval=5",
expected: 5 * time.Millisecond,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?config_poll_interval=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.ConfigPollerConfig.CccpPollPeriod != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.ConfigPollerConfig.CccpPollPeriod)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_EnableMutationTokens() {
tests := []struct {
name string
connStr string
expected bool
wantErr bool
}{
{
name: "true",
connStr: "couchbase://10.112.192.101?enable_mutation_tokens=true",
expected: true,
},
{
name: "false",
connStr: "couchbase://10.112.192.101?enable_mutation_tokens=false",
expected: false,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?enable_mutation_tokens=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.IoConfig.UseMutationTokens != tt.expected {
suite.T().Fatalf("Expected %t but was %t", tt.expected, config.IoConfig.UseMutationTokens)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_Compression() {
tests := []struct {
name string
connStr string
expected bool
wantErr bool
}{
{
name: "true",
connStr: "couchbase://10.112.192.101?compression=true",
expected: true,
},
{
name: "false",
connStr: "couchbase://10.112.192.101?compression=false",
expected: false,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?compression=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.CompressionConfig.Enabled != tt.expected {
suite.T().Fatalf("Expected %t but was %t", tt.expected, config.CompressionConfig.Enabled)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_CompressionMinSize() {
tests := []struct {
name string
connStr string
expected int
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?compression_min_size=100000",
expected: 100000,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?compression_min_size=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.CompressionConfig.MinSize != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.CompressionConfig.MinSize)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_CompressionMinRatio() {
tests := []struct {
name string
connStr string
expected float64
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?compression_min_ratio=0.7",
expected: 0.7,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?compression_min_ratio=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.CompressionConfig.MinRatio != tt.expected {
suite.T().Fatalf("Expected %f but was %f", tt.expected, config.CompressionConfig.MinRatio)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_ServerDurations() {
tests := []struct {
name string
connStr string
expected bool
wantErr bool
}{
{
name: "true",
connStr: "couchbase://10.112.192.101?enable_server_durations=true",
expected: true,
},
{
name: "false",
connStr: "couchbase://10.112.192.101?enable_server_durations=false",
expected: false,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?enable_server_durations=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.IoConfig.UseDurations != tt.expected {
suite.T().Fatalf("Expected %t but was %t", tt.expected, config.IoConfig.UseDurations)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_MaxIdleHTTPConnections() {
tests := []struct {
name string
connStr string
expected int
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?max_idle_http_connections=2",
expected: 2,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?max_idle_http_connections=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.HTTPConfig.MaxIdleConns != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.HTTPConfig.MaxIdleConns)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_MaxPerHostIdleHTTPConnections() {
tests := []struct {
name string
connStr string
expected int
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?max_perhost_idle_http_connections=2",
expected: 2,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?max_perhost_idle_http_connections=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.HTTPConfig.MaxIdleConnsPerHost != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.HTTPConfig.MaxIdleConnsPerHost)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_IdleHTTPConnectionTimeout() {
tests := []struct {
name string
connStr string
expected time.Duration
wantErr bool
}{
{
name: "duration",
connStr: "couchbase://10.112.192.101?idle_http_connection_timeout=5000us",
expected: 5 * time.Millisecond,
},
{
name: "ms",
connStr: "couchbase://10.112.192.101?idle_http_connection_timeout=5",
expected: 5 * time.Millisecond,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?idle_http_connection_timeout=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.HTTPConfig.IdleConnectionTimeout != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.HTTPConfig.IdleConnectionTimeout)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_OrphanResponseLogging() {
tests := []struct {
name string
connStr string
expected bool
wantErr bool
}{
{
name: "true",
connStr: "couchbase://10.112.192.101?orphaned_response_logging=true",
expected: true,
},
{
name: "false",
connStr: "couchbase://10.112.192.101?orphaned_response_logging=false",
expected: false,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?orphaned_response_logging=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.OrphanReporterConfig.Enabled != tt.expected {
suite.T().Fatalf("Expected %t but was %t", tt.expected, config.OrphanReporterConfig.Enabled)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_OrphanResponseLoggingInterval() {
tests := []struct {
name string
connStr string
expected time.Duration
wantErr bool
}{
{
name: "duration",
connStr: "couchbase://10.112.192.101?orphaned_response_logging_interval=5000us",
expected: 5 * time.Millisecond,
},
{
name: "ms",
connStr: "couchbase://10.112.192.101?orphaned_response_logging_interval=5",
expected: 5 * time.Millisecond,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?orphaned_response_logging_interval=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.OrphanReporterConfig.ReportInterval != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.OrphanReporterConfig.ReportInterval)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_OrphanResponseLoggerSampleSize() {
tests := []struct {
name string
connStr string
expected int
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?orphaned_response_logging_sample_size=2",
expected: 2,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?orphaned_response_logging_sample_size=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.OrphanReporterConfig.SampleSize != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.OrphanReporterConfig.SampleSize)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_HTTPRedialPeriod() {
tests := []struct {
name string
connStr string
expected time.Duration
wantErr bool
}{
{
name: "duration",
connStr: "couchbase://10.112.192.101?http_redial_period=5000us",
expected: 5 * time.Millisecond,
},
{
name: "ms",
connStr: "couchbase://10.112.192.101?http_redial_period=5",
expected: 5 * time.Millisecond,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?http_redial_period=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.ConfigPollerConfig.HTTPRedialPeriod != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.ConfigPollerConfig.HTTPRedialPeriod)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_HTTPRetryDelay() {
tests := []struct {
name string
connStr string
expected time.Duration
wantErr bool
}{
{
name: "duration",
connStr: "couchbase://10.112.192.101?http_retry_delay=5000us",
expected: 5 * time.Millisecond,
},
{
name: "ms",
connStr: "couchbase://10.112.192.101?http_retry_delay=5",
expected: 5 * time.Millisecond,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?http_retry_delay=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.ConfigPollerConfig.HTTPRetryDelay != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.ConfigPollerConfig.HTTPRetryDelay)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_KVPoolSize() {
tests := []struct {
name string
connStr string
expected int
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?kv_pool_size=2",
expected: 2,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?kv_pool_size=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.KVConfig.PoolSize != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.KVConfig.PoolSize)
}
})
}
}
func (suite *StandardTestSuite) TestAgentConfig_MaxQueueSize() {
tests := []struct {
name string
connStr string
expected int
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?max_queue_size=2",
expected: 2,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?max_queue_size=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &AgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.KVConfig.MaxQueueSize != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.KVConfig.MaxQueueSize)
}
})
}
}
gocbcore-10.2.3/agent_internal.go 0000664 0000000 0000000 00000001142 14417540156 0016706 0 ustar 00root root 0000000 0000000 package gocbcore
// AgentInternal is a set of internal only functionality.
// Internal: This should never be used and is not supported.
type AgentInternal struct {
agent *Agent
}
// Internal creates a new AgentInternal.
// Internal: This should never be used and is not supported.
func (agent *Agent) Internal() *AgentInternal {
return &AgentInternal{
agent: agent,
}
}
// BucketCapabilityStatus returns the current status for a given bucket capability.
func (ai *AgentInternal) BucketCapabilityStatus(cap BucketCapability) BucketCapabilityStatus {
return ai.agent.kvMux.BucketCapabilityStatus(cap)
}
gocbcore-10.2.3/agent_internal_test.go 0000664 0000000 0000000 00000001211 14417540156 0017742 0 ustar 00root root 0000000 0000000 package gocbcore
func (suite *StandardTestSuite) TestInternalBucketCapabilityStatus() {
internal := suite.DefaultAgent().Internal()
agent := suite.DefaultAgent()
state := agent.kvMux.getState()
suite.Assert().Equal(state.bucketCapabilities[BucketCapabilityReplaceBodyWithXattr], internal.BucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr))
suite.Assert().Equal(state.bucketCapabilities[BucketCapabilityCreateAsDeleted], internal.BucketCapabilityStatus(BucketCapabilityCreateAsDeleted))
suite.Assert().Equal(state.bucketCapabilities[BucketCapabilityDurableWrites], internal.BucketCapabilityStatus(BucketCapabilityDurableWrites))
}
gocbcore-10.2.3/agent_ops.go 0000664 0000000 0000000 00000034156 14417540156 0015706 0 ustar 00root root 0000000 0000000 package gocbcore
import "github.com/couchbase/gocbcore/v10/memd"
// GetCallback is invoked upon completion of a Get operation.
type GetCallback func(*GetResult, error)
// Get retrieves a document.
func (agent *Agent) Get(opts GetOptions, cb GetCallback) (PendingOp, error) {
return agent.crud.Get(opts, cb)
}
// GetAndTouchCallback is invoked upon completion of a GetAndTouch operation.
type GetAndTouchCallback func(*GetAndTouchResult, error)
// GetAndTouch retrieves a document and updates its expiry.
func (agent *Agent) GetAndTouch(opts GetAndTouchOptions, cb GetAndTouchCallback) (PendingOp, error) {
return agent.crud.GetAndTouch(opts, cb)
}
// GetAndLockCallback is invoked upon completion of a GetAndLock operation.
type GetAndLockCallback func(*GetAndLockResult, error)
// GetAndLock retrieves a document and locks it.
func (agent *Agent) GetAndLock(opts GetAndLockOptions, cb GetAndLockCallback) (PendingOp, error) {
return agent.crud.GetAndLock(opts, cb)
}
// GetReplicaCallback is invoked upon completion of a GetReplica operation.
type GetReplicaCallback func(*GetReplicaResult, error)
// GetOneReplica retrieves a document from a replica server.
func (agent *Agent) GetOneReplica(opts GetOneReplicaOptions, cb GetReplicaCallback) (PendingOp, error) {
return agent.crud.GetOneReplica(opts, cb)
}
// TouchCallback is invoked upon completion of a Touch operation.
type TouchCallback func(*TouchResult, error)
// Touch updates the expiry for a document.
func (agent *Agent) Touch(opts TouchOptions, cb TouchCallback) (PendingOp, error) {
return agent.crud.Touch(opts, cb)
}
// UnlockCallback is invoked upon completion of a Unlock operation.
type UnlockCallback func(*UnlockResult, error)
// Unlock unlocks a locked document.
func (agent *Agent) Unlock(opts UnlockOptions, cb UnlockCallback) (PendingOp, error) {
return agent.crud.Unlock(opts, cb)
}
// DeleteCallback is invoked upon completion of a Delete operation.
type DeleteCallback func(*DeleteResult, error)
// Delete removes a document.
func (agent *Agent) Delete(opts DeleteOptions, cb DeleteCallback) (PendingOp, error) {
return agent.crud.Delete(opts, cb)
}
// StoreCallback is invoked upon completion of a Add, Set or Replace operation.
type StoreCallback func(*StoreResult, error)
// Add stores a document as long as it does not already exist.
func (agent *Agent) Add(opts AddOptions, cb StoreCallback) (PendingOp, error) {
return agent.crud.Add(opts, cb)
}
// Set stores a document.
func (agent *Agent) Set(opts SetOptions, cb StoreCallback) (PendingOp, error) {
return agent.crud.Set(opts, cb)
}
// Replace replaces the value of a Couchbase document with another value.
func (agent *Agent) Replace(opts ReplaceOptions, cb StoreCallback) (PendingOp, error) {
return agent.crud.Replace(opts, cb)
}
// AdjoinCallback is invoked upon completion of a Append or Prepend operation.
type AdjoinCallback func(*AdjoinResult, error)
// Append appends some bytes to a document.
func (agent *Agent) Append(opts AdjoinOptions, cb AdjoinCallback) (PendingOp, error) {
return agent.crud.Append(opts, cb)
}
// Prepend prepends some bytes to a document.
func (agent *Agent) Prepend(opts AdjoinOptions, cb AdjoinCallback) (PendingOp, error) {
return agent.crud.Prepend(opts, cb)
}
// CounterCallback is invoked upon completion of a Increment or Decrement operation.
type CounterCallback func(*CounterResult, error)
// Increment increments the unsigned integer value in a document.
func (agent *Agent) Increment(opts CounterOptions, cb CounterCallback) (PendingOp, error) {
return agent.crud.Increment(opts, cb)
}
// Decrement decrements the unsigned integer value in a document.
func (agent *Agent) Decrement(opts CounterOptions, cb CounterCallback) (PendingOp, error) {
return agent.crud.Decrement(opts, cb)
}
// GetRandomCallback is invoked upon completion of a GetRandom operation.
type GetRandomCallback func(*GetRandomResult, error)
// GetRandom retrieves the key and value of a random document stored within Couchbase Server.
func (agent *Agent) GetRandom(opts GetRandomOptions, cb GetRandomCallback) (PendingOp, error) {
return agent.crud.GetRandom(opts, cb)
}
// GetMetaCallback is invoked upon completion of a GetMeta operation.
type GetMetaCallback func(*GetMetaResult, error)
// GetMeta retrieves a document along with some internal Couchbase meta-data.
func (agent *Agent) GetMeta(opts GetMetaOptions, cb GetMetaCallback) (PendingOp, error) {
return agent.crud.GetMeta(opts, cb)
}
// SetMetaCallback is invoked upon completion of a SetMeta operation.
type SetMetaCallback func(*SetMetaResult, error)
// SetMeta stores a document along with setting some internal Couchbase meta-data.
func (agent *Agent) SetMeta(opts SetMetaOptions, cb SetMetaCallback) (PendingOp, error) {
return agent.crud.SetMeta(opts, cb)
}
// DeleteMetaCallback is invoked upon completion of a DeleteMeta operation.
type DeleteMetaCallback func(*DeleteMetaResult, error)
// DeleteMeta deletes a document along with setting some internal Couchbase meta-data.
func (agent *Agent) DeleteMeta(opts DeleteMetaOptions, cb DeleteMetaCallback) (PendingOp, error) {
return agent.crud.DeleteMeta(opts, cb)
}
// StatsCallback is invoked upon completion of a Stats operation.
type StatsCallback func(*StatsResult, error)
// Stats retrieves statistics information from the server. Note that as this
// function is an aggregator across numerous servers, there are no guarantees
// about the consistency of the results. Occasionally, some nodes may not be
// represented in the results, or there may be conflicting information between
// multiple nodes (a vbucket active on two separate nodes at once).
func (agent *Agent) Stats(opts StatsOptions, cb StatsCallback) (PendingOp, error) {
return agent.stats.Stats(opts, cb)
}
// ObserveCallback is invoked upon completion of a Observe operation.
type ObserveCallback func(*ObserveResult, error)
// Observe retrieves the current CAS and persistence state for a document.
func (agent *Agent) Observe(opts ObserveOptions, cb ObserveCallback) (PendingOp, error) {
return agent.observe.Observe(opts, cb)
}
// ObserveVbCallback is invoked upon completion of a ObserveVb operation.
type ObserveVbCallback func(*ObserveVbResult, error)
// ObserveVb retrieves the persistence state sequence numbers for a particular VBucket
// and includes additional details not included by the basic version.
func (agent *Agent) ObserveVb(opts ObserveVbOptions, cb ObserveVbCallback) (PendingOp, error) {
return agent.observe.ObserveVb(opts, cb)
}
// SubDocOp defines a per-operation structure to be passed to MutateIn
// or LookupIn for performing many sub-document operations.
type SubDocOp struct {
Op memd.SubDocOpType
Flags memd.SubdocFlag
Path string
Value []byte
}
// LookupInCallback is invoked upon completion of a LookupIn operation.
type LookupInCallback func(*LookupInResult, error)
// LookupIn performs a multiple-lookup sub-document operation on a document.
func (agent *Agent) LookupIn(opts LookupInOptions, cb LookupInCallback) (PendingOp, error) {
return agent.crud.LookupIn(opts, cb)
}
// MutateInCallback is invoked upon completion of a MutateIn operation.
type MutateInCallback func(*MutateInResult, error)
// MutateIn performs a multiple-mutation sub-document operation on a document.
func (agent *Agent) MutateIn(opts MutateInOptions, cb MutateInCallback) (PendingOp, error) {
return agent.crud.MutateIn(opts, cb)
}
// N1QLQueryCallback is invoked upon completion of a N1QLQuery operation.
type N1QLQueryCallback func(*N1QLRowReader, error)
// N1QLQuery executes a N1QL query
func (agent *Agent) N1QLQuery(opts N1QLQueryOptions, cb N1QLQueryCallback) (PendingOp, error) {
return agent.n1ql.N1QLQuery(opts, cb)
}
// PreparedN1QLQuery executes a prepared N1QL query
func (agent *Agent) PreparedN1QLQuery(opts N1QLQueryOptions, cb N1QLQueryCallback) (PendingOp, error) {
return agent.n1ql.PreparedN1QLQuery(opts, cb)
}
// AnalyticsQueryCallback is invoked upon completion of a AnalyticsQuery operation.
type AnalyticsQueryCallback func(*AnalyticsRowReader, error)
// AnalyticsQuery executes an analytics query
func (agent *Agent) AnalyticsQuery(opts AnalyticsQueryOptions, cb AnalyticsQueryCallback) (PendingOp, error) {
return agent.analytics.AnalyticsQuery(opts, cb)
}
// SearchQueryCallback is invoked upon completion of a SearchQuery operation.
type SearchQueryCallback func(*SearchRowReader, error)
// SearchQuery executes a Search query
func (agent *Agent) SearchQuery(opts SearchQueryOptions, cb SearchQueryCallback) (PendingOp, error) {
return agent.search.SearchQuery(opts, cb)
}
// ViewQueryCallback is invoked upon completion of a ViewQuery operation.
type ViewQueryCallback func(*ViewQueryRowReader, error)
// ViewQuery executes a view query
func (agent *Agent) ViewQuery(opts ViewQueryOptions, cb ViewQueryCallback) (PendingOp, error) {
return agent.views.ViewQuery(opts, cb)
}
// DoHTTPRequestCallback is invoked upon completion of a DoHTTPRequest operation.
type DoHTTPRequestCallback func(*HTTPResponse, error)
// DoHTTPRequest will perform an HTTP request against one of the HTTP
// services which are available within the SDK.
func (agent *Agent) DoHTTPRequest(req *HTTPRequest, cb DoHTTPRequestCallback) (PendingOp, error) {
return agent.http.DoHTTPRequest(req, cb)
}
// GetCollectionManifestCallback is invoked upon completion of a GetCollectionManifest operation.
type GetCollectionManifestCallback func(*GetCollectionManifestResult, error)
// GetCollectionManifest fetches the current server manifest. This function will not update the client's collection
// id cache.
func (agent *Agent) GetCollectionManifest(opts GetCollectionManifestOptions, cb GetCollectionManifestCallback) (PendingOp, error) {
return agent.collections.GetCollectionManifest(opts, cb)
}
// GetAllCollectionManifestsCallback is invoked upon completion of a GetAllCollectionManifests operation.
type GetAllCollectionManifestsCallback func(*GetAllCollectionManifestsResult, error)
// GetAllCollectionManifests fetches the collection manifest from each server in the cluster, note that it's possible
// for one or mode nodes to be slightly behind when creating scopes/collections. This function will not update the
// client's collection id cache.
func (agent *Agent) GetAllCollectionManifests(opts GetAllCollectionManifestsOptions,
cb GetAllCollectionManifestsCallback) (PendingOp, error) {
return agent.collections.GetAllCollectionManifests(opts, cb)
}
// GetCollectionIDCallback is invoked upon completion of a GetCollectionID operation.
type GetCollectionIDCallback func(*GetCollectionIDResult, error)
// GetCollectionID fetches the collection id and manifest id that the collection belongs to, given a scope name
// and collection name. This function will also prime the client's collection id cache.
func (agent *Agent) GetCollectionID(scopeName string, collectionName string, opts GetCollectionIDOptions, cb GetCollectionIDCallback) (PendingOp, error) {
return agent.collections.GetCollectionID(scopeName, collectionName, opts, cb)
}
// PingCallback is invoked upon completion of a PingKv operation.
type PingCallback func(*PingResult, error)
// Ping pings all of the servers we are connected to and returns
// a report regarding the pings that were performed.
func (agent *Agent) Ping(opts PingOptions, cb PingCallback) (PendingOp, error) {
return agent.diagnostics.Ping(opts, cb)
}
// Diagnostics returns diagnostics information about the client.
// Mainly containing a list of open connections and their current
// states.
func (agent *Agent) Diagnostics(opts DiagnosticsOptions) (*DiagnosticInfo, error) {
return agent.diagnostics.Diagnostics(opts)
}
// WaitUntilReadyCallback is invoked upon completion of a WaitUntilReady operation.
type WaitUntilReadyCallback func(*WaitUntilReadyResult, error)
// RangeScanCreateCallback is invoked upon completion of a RangeScanCreate operation.
// Volatile: This API is subject to change at any time.
type RangeScanCreateCallback func(*RangeScanCreateResult, error)
// RangeScanCreate creates a new range scan against a vbucket.
// Volatile: This API is subject to change at any time.
func (agent *Agent) RangeScanCreate(vbID uint16, opts RangeScanCreateOptions, cb RangeScanCreateCallback) (PendingOp, error) {
return agent.crud.RangeScanCreate(vbID, opts, cb)
}
// RangeScanContinueDataCallback is invoked upon receipt of a RangeScanContinue response containing data.
// Volatile: This API is subject to change at any time.
type RangeScanContinueDataCallback func([]RangeScanItem)
// RangeScanContinueActionCallback is invoked upon receipt of a RangeScanContinue response representing an action.
// Volatile: This API is subject to change at any time.
type RangeScanContinueActionCallback func(*RangeScanContinueResult, error)
// RangeScanContinue continues an existing range scan against a vbucket.
// Volatile: This API is subject to change at any time.
func (agent *Agent) RangeScanContinue(scanUUID []byte, vbID uint16, opts RangeScanContinueOptions, dataCb RangeScanContinueDataCallback,
eventCb RangeScanContinueActionCallback) (PendingOp, error) {
return agent.crud.RangeScanContinue(scanUUID, vbID, opts, dataCb, eventCb)
}
// RangeScanCancelCallback is invoked upon completion of a RangeScanCancel operation.
// Volatile: This API is subject to change at any time.
type RangeScanCancelCallback func(*RangeScanCancelResult, error)
// RangeScanCancel cancels an existing range scan against a vbucket.
// Volatile: This API is subject to change at any time.
func (agent *Agent) RangeScanCancel(scanUUID []byte, vbID uint16, opts RangeScanCancelOptions, cb RangeScanCancelCallback) (PendingOp, error) {
return agent.crud.RangeScanCancel(scanUUID, vbID, opts, cb)
}
// WaitForConfigSnapshotOptions encapsulates the parameters for a WaitForConfigSnapshot operation.
// Volatile: This API is subject to change at any time.
type WaitForConfigSnapshotOptions struct {
}
// WaitForConfigSnapshotResult encapsulates the result of a WaitForConfig operation.
// Volatile: This API is subject to change at any time.
type WaitForConfigSnapshotResult struct {
Snapshot *ConfigSnapshot
}
// WaitForConfigSnapshotCallback is invoked upon completion of a WaitForConfigSnapshot operation.
// Volatile: This API is subject to change at any time.
type WaitForConfigSnapshotCallback func(*WaitForConfigSnapshotResult, error)
gocbcore-10.2.3/agent_test.go 0000664 0000000 0000000 00000264564 14417540156 0016074 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"bytes"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/url"
"strconv"
"strings"
"testing"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func (suite *StandardTestSuite) TestCidRetries() {
suite.EnsureSupportsFeature(TestFeatureCollections)
agent, s := suite.GetAgentAndHarness()
bucketName := suite.BucketName
scopeName := suite.ScopeName
collectionName := "testCidRetries"
_, err := testCreateCollection(collectionName, scopeName, bucketName, agent)
if err != nil {
suite.T().Logf("Failed to create collection: %v", err)
}
// prime the cid map cache
s.PushOp(agent.GetCollectionID(scopeName, collectionName, GetCollectionIDOptions{},
func(result *GetCollectionIDResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get CID operation failed: %v", err)
}
})
}),
)
s.Wait(0)
// delete the collection
_, err = testDeleteCollection(collectionName, scopeName, bucketName, agent, true)
if err != nil {
suite.T().Fatalf("Failed to delete collection: %v", err)
}
// recreate
_, err = testCreateCollection(collectionName, scopeName, bucketName, agent)
if err != nil {
suite.T().Fatalf("Failed to create collection: %v", err)
}
// Set should succeed as we detect cid unknown, fetch the cid and then retry again. This should happen
// even if we don't set a retry strategy.
s.PushOp(agent.Set(SetOptions{
Key: []byte("test"),
Value: []byte("{}"),
CollectionName: collectionName,
ScopeName: scopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
// Get
s.PushOp(agent.Get(GetOptions{
Key: []byte("test"),
CollectionName: collectionName,
ScopeName: scopeName,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
}
func (suite *StandardTestSuite) TestPreserveExpirySet() {
suite.EnsureSupportsFeature(TestFeaturePreserveExpiry)
agent, s := suite.GetAgentAndHarness()
expiry := uint32(25)
// Set
s.PushOp(agent.Set(SetOptions{
Key: []byte("testsetpreserveExpiry"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
Expiry: expiry,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
s.PushOp(agent.Set(SetOptions{
Key: []byte("testsetpreserveExpiry"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
PreserveExpiry: true,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
// Get
s.PushOp(agent.GetMeta(GetMetaOptions{
Key: []byte("testsetpreserveExpiry"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetMetaResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("GetMeta operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
expectedExpiry := uint32(time.Now().Unix() + int64(expiry-5))
if res.Expiry < expectedExpiry {
s.Fatalf("Invalid expiry received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(3, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testsetpreserveExpiry")
suite.AssertOpSpan(nilParents[1], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testsetpreserveExpiry")
suite.AssertOpSpan(nilParents[2], "GetMeta", agent.BucketName(), memd.CmdGetMeta.Name(), 1, false, "testsetpreserveExpiry")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 2, false, false)
suite.VerifyKVMetrics(suite.meter, "GetMeta", 1, false, false)
}
func (suite *StandardTestSuite) TestPreserveExpiryReplace() {
suite.EnsureSupportsFeature(TestFeaturePreserveExpiry)
agent, s := suite.GetAgentAndHarness()
expiry := uint32(25)
// Set
s.PushOp(agent.Set(SetOptions{
Key: []byte("testreplacepreserveExpiry"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
Expiry: expiry,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
s.PushOp(agent.Replace(ReplaceOptions{
Key: []byte("testreplacepreserveExpiry"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
PreserveExpiry: true,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Replace operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
// Get
s.PushOp(agent.GetMeta(GetMetaOptions{
Key: []byte("testreplacepreserveExpiry"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetMetaResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("GetMeta operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
expectedExpiry := uint32(time.Now().Unix() + int64(expiry-5))
if res.Expiry < expectedExpiry {
s.Fatalf("Invalid expiry received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(3, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testreplacepreserveExpiry")
suite.AssertOpSpan(nilParents[1], "Replace", agent.BucketName(), memd.CmdReplace.Name(), 1, false, "testreplacepreserveExpiry")
suite.AssertOpSpan(nilParents[2], "GetMeta", agent.BucketName(), memd.CmdGetMeta.Name(), 1, false, "testreplacepreserveExpiry")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Replace", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "GetMeta", 1, false, false)
}
func (suite *StandardTestSuite) TestPreserveExpiryAppend() {
suite.EnsureSupportsFeature(TestFeaturePreserveExpiry)
agent, s := suite.GetAgentAndHarness()
expiry := uint32(25)
// Set
s.PushOp(agent.Set(SetOptions{
Key: []byte("testappendpreserveExpiry"),
Value: []byte("hello "),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
Expiry: expiry,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
s.PushOp(agent.Append(AdjoinOptions{
Key: []byte("testappendpreserveExpiry"),
Value: []byte("world"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
PreserveExpiry: true,
}, func(res *AdjoinResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Replace operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
// Get
s.PushOp(agent.GetMeta(GetMetaOptions{
Key: []byte("testappendpreserveExpiry"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetMetaResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("GetMeta operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
expectedExpiry := uint32(time.Now().Unix() + int64(expiry-5))
if res.Expiry < expectedExpiry {
s.Fatalf("Invalid expiry received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(3, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testappendpreserveExpiry")
suite.AssertOpSpan(nilParents[1], "Append", agent.BucketName(), memd.CmdAppend.Name(), 1, false, "testappendpreserveExpiry")
suite.AssertOpSpan(nilParents[2], "GetMeta", agent.BucketName(), memd.CmdGetMeta.Name(), 1, false, "testappendpreserveExpiry")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Append", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "GetMeta", 1, false, false)
}
func (suite *StandardTestSuite) TestPreserveExpiryIncrement() {
suite.EnsureSupportsFeature(TestFeaturePreserveExpiry)
agent, s := suite.GetAgentAndHarness()
expiry := uint32(25)
s.PushOp(agent.Increment(CounterOptions{
Key: []byte("testincrementpreserveExpiry"),
Initial: 5,
Delta: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
PreserveExpiry: true,
Expiry: expiry,
}, func(res *CounterResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Replace operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
s.PushOp(agent.Increment(CounterOptions{
Key: []byte("testincrementpreserveExpiry"),
Delta: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
PreserveExpiry: true,
}, func(res *CounterResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Replace operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
// Get
s.PushOp(agent.GetMeta(GetMetaOptions{
Key: []byte("testincrementpreserveExpiry"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetMetaResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("GetMeta operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
expectedExpiry := uint32(time.Now().Unix() + int64(expiry-5))
if res.Expiry < expectedExpiry {
s.Fatalf("Invalid expiry received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(3, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Increment", agent.BucketName(), memd.CmdIncrement.Name(), 1, false, "testincrementpreserveExpiry")
suite.AssertOpSpan(nilParents[2], "GetMeta", agent.BucketName(), memd.CmdGetMeta.Name(), 1, false, "testincrementpreserveExpiry")
}
}
suite.VerifyKVMetrics(suite.meter, "Increment", 2, false, false)
suite.VerifyKVMetrics(suite.meter, "GetMeta", 1, false, false)
}
func (suite *StandardTestSuite) TestBasicOps() {
agent, s := suite.GetAgentAndHarness()
// Set
s.PushOp(agent.Set(SetOptions{
Key: []byte("test"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
// Get
s.PushOp(agent.Get(GetOptions{
Key: []byte("test"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(2, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "test")
suite.AssertOpSpan(nilParents[1], "Get", agent.BucketName(), memd.CmdGet.Name(), 1, false, "test")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Get", 1, false, false)
}
func (suite *StandardTestSuite) TestCasMismatch() {
agent, s := suite.GetAgentAndHarness()
// Set
var cas Cas
s.PushOp(agent.Set(SetOptions{
Key: []byte("testCasMismatch"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
cas = res.Cas
})
}))
s.Wait(0)
// Replace to change cas on the server
s.PushOp(agent.Replace(ReplaceOptions{
Key: []byte("testCasMismatch"),
Value: []byte("{\"key\":\"value\"}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Replace operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
// Replace which should fail with a cas mismatch
s.PushOp(agent.Replace(ReplaceOptions{
Key: []byte("testCasMismatch"),
Value: []byte("{\"key\":\"value2\"}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
Cas: cas,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err == nil {
s.Fatalf("Set operation succeeded but should have failed")
}
if !errors.Is(err, ErrCasMismatch) {
suite.T().Fatalf("Expected CasMismatch error but was %v", err)
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(3, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testCasMismatch")
suite.AssertOpSpan(nilParents[1], "Replace", agent.BucketName(), memd.CmdReplace.Name(), 1, false, "testCasMismatch")
suite.AssertOpSpan(nilParents[2], "Replace", agent.BucketName(), memd.CmdReplace.Name(), 1, false, "testCasMismatch")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Replace", 2, false, false)
}
func (suite *StandardTestSuite) TestGetReplica() {
suite.EnsureSupportsFeature(TestFeatureReplicas)
agent, s := suite.GetAgentAndHarness()
// Set
s.PushOp(agent.Set(SetOptions{
Key: []byte("testReplica"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
retries := 0
keyExists := false
for {
s.PushOp(agent.GetOneReplica(GetOneReplicaOptions{
Key: []byte("testReplica"),
ReplicaIdx: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetReplicaResult, err error) {
s.Wrap(func() {
keyNotFound := errors.Is(err, ErrDocumentNotFound)
if err == nil {
keyExists = true
} else if err != nil && !keyNotFound {
s.Fatalf("GetReplica specific returned error that was not document not found: %v", err)
}
if !keyNotFound && res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if keyExists {
break
}
retries++
if retries >= 5 {
suite.T().Fatalf("GetReplica could not locate key")
}
time.Sleep(50 * time.Millisecond)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 2) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testReplica")
suite.AssertOpSpan(nilParents[1], "GetOneReplica", agent.BucketName(), memd.CmdGetReplica.Name(), 1, true, "testReplica")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "GetOneReplica", 1, true, false)
}
func (suite *StandardTestSuite) TestDurableWriteGetReplica() {
suite.EnsureSupportsFeature(TestFeatureReplicas)
suite.EnsureSupportsFeature(TestFeatureEnhancedDurability)
agent, s := suite.GetAgentAndHarness()
// Set
s.PushOp(agent.Set(SetOptions{
Key: []byte("testDurableReplica"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
DurabilityLevel: memd.DurabilityLevelMajority,
DurabilityLevelTimeout: 10 * time.Second,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
retries := 0
keyExists := false
for {
s.PushOp(agent.GetOneReplica(GetOneReplicaOptions{
Key: []byte("testDurableReplica"),
ReplicaIdx: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetReplicaResult, err error) {
s.Wrap(func() {
keyNotFound := errors.Is(err, ErrDocumentNotFound)
if err == nil {
keyExists = true
} else if err != nil && !keyNotFound {
s.Fatalf("GetReplica specific returned error that was not document not found: %v", err)
}
if !keyNotFound && res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if keyExists {
break
}
retries++
if retries >= 5 {
suite.T().Fatalf("GetReplica could not locate key")
}
time.Sleep(50 * time.Millisecond)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 2) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testDurableReplica")
suite.AssertOpSpan(nilParents[1], "GetOneReplica", agent.BucketName(), memd.CmdGetReplica.Name(), 1, true, "testDurableReplica")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "GetOneReplica", 1, true, false)
}
func (suite *StandardTestSuite) TestAddDurableWriteGetReplica() {
suite.EnsureSupportsFeature(TestFeatureReplicas)
suite.EnsureSupportsFeature(TestFeatureEnhancedDurability)
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Add(AddOptions{
Key: []byte("testAddDurableReplica"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
DurabilityLevel: memd.DurabilityLevelMajority,
DurabilityLevelTimeout: 10 * time.Second,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Add operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
retries := 0
keyExists := false
for {
s.PushOp(agent.GetOneReplica(GetOneReplicaOptions{
Key: []byte("testAddDurableReplica"),
ReplicaIdx: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetReplicaResult, err error) {
s.Wrap(func() {
keyNotFound := errors.Is(err, ErrDocumentNotFound)
if err == nil {
keyExists = true
} else if err != nil && !keyNotFound {
s.Fatalf("GetReplica specific returned error that was not document not found: %v", err)
}
if !keyNotFound && res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if keyExists {
break
}
retries++
if retries >= 5 {
suite.T().Fatalf("GetReplica could not locate key")
}
time.Sleep(50 * time.Millisecond)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 2) {
suite.AssertOpSpan(nilParents[0], "Add", agent.BucketName(), memd.CmdAdd.Name(), 1, false, "testAddDurableReplica")
suite.AssertOpSpan(nilParents[1], "GetOneReplica", agent.BucketName(), memd.CmdGetReplica.Name(), 1, true, "testAddDurableReplica")
}
}
suite.VerifyKVMetrics(suite.meter, "Add", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "GetOneReplica", 1, true, false)
}
func (suite *StandardTestSuite) TestReplaceDurableWriteGetReplica() {
suite.EnsureSupportsFeature(TestFeatureReplicas)
suite.EnsureSupportsFeature(TestFeatureEnhancedDurability)
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testReplaceDurableReplica"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
DurabilityLevel: memd.DurabilityLevelMajority,
DurabilityLevelTimeout: 10 * time.Second,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
s.PushOp(agent.Replace(ReplaceOptions{
Key: []byte("testReplaceDurableReplica"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
DurabilityLevel: memd.DurabilityLevelMajority,
DurabilityLevelTimeout: 10 * time.Second,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Replace operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
retries := 0
keyExists := false
for {
s.PushOp(agent.GetOneReplica(GetOneReplicaOptions{
Key: []byte("testReplaceDurableReplica"),
ReplicaIdx: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetReplicaResult, err error) {
s.Wrap(func() {
keyNotFound := errors.Is(err, ErrDocumentNotFound)
if err == nil {
keyExists = true
} else if err != nil && !keyNotFound {
s.Fatalf("GetReplica specific returned error that was not document not found: %v", err)
}
if !keyNotFound && res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if keyExists {
break
}
retries++
if retries >= 5 {
suite.T().Fatalf("GetReplica could not locate key")
}
time.Sleep(50 * time.Millisecond)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 3) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testReplaceDurableReplica")
suite.AssertOpSpan(nilParents[1], "Replace", agent.BucketName(), memd.CmdReplace.Name(), 1, false, "testReplaceDurableReplica")
suite.AssertOpSpan(nilParents[2], "GetOneReplica", agent.BucketName(), memd.CmdGetReplica.Name(), 1, true, "testReplaceDurableReplica")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Replace", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "GetOneReplica", 1, true, false)
}
func (suite *StandardTestSuite) TestDeleteDurableWriteGetReplica() {
suite.EnsureSupportsFeature(TestFeatureReplicas)
suite.EnsureSupportsFeature(TestFeatureEnhancedDurability)
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testDeleteDurableReplica"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
DurabilityLevel: memd.DurabilityLevelMajority,
DurabilityLevelTimeout: 10 * time.Second,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
s.PushOp(agent.Delete(DeleteOptions{
Key: []byte("testDeleteDurableReplica"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
DurabilityLevel: memd.DurabilityLevelMajority,
DurabilityLevelTimeout: 10 * time.Second,
}, func(res *DeleteResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Delete operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
retries := 0
keyNotFound := false
for {
s.PushOp(agent.GetOneReplica(GetOneReplicaOptions{
Key: []byte("testDeleteDurableReplica"),
ReplicaIdx: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetReplicaResult, err error) {
s.Wrap(func() {
if errors.Is(err, ErrDocumentNotFound) {
keyNotFound = true
} else if err != nil {
s.Fatalf("GetReplica specific returned error that was not document not found: %v", err)
}
if !keyNotFound && res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if keyNotFound {
break
}
retries++
if retries >= 5 {
suite.T().Fatalf("GetReplica could always locate key")
}
time.Sleep(50 * time.Millisecond)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 3) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testDeleteDurableReplica")
suite.AssertOpSpan(nilParents[1], "Delete", agent.BucketName(), memd.CmdDelete.Name(), 1, false, "testDeleteDurableReplica")
suite.AssertOpSpan(nilParents[2], "GetOneReplica", agent.BucketName(), memd.CmdGetReplica.Name(), 1, true, "testDeleteDurableReplica")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Delete", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "GetOneReplica", 1, true, false)
}
func (suite *StandardTestSuite) TestBasicReplace() {
agent, s := suite.GetAgentAndHarness()
oldCas := Cas(0)
s.PushOp(agent.Set(SetOptions{
Key: []byte("testx"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
oldCas = res.Cas
s.Continue()
}))
s.Wait(0)
s.PushOp(agent.Replace(ReplaceOptions{
Key: []byte("testx"),
Value: []byte("[]"),
Cas: oldCas,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Replace operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(2, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testx")
suite.AssertOpSpan(nilParents[1], "Replace", agent.BucketName(), memd.CmdReplace.Name(), 1, false, "testx")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Replace", 1, false, false)
}
func (suite *StandardTestSuite) TestBasicRemove() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testy"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Continue()
}))
s.Wait(0)
s.PushOp(agent.Delete(DeleteOptions{
Key: []byte("testy"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *DeleteResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Remove operation failed: %v", err)
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(2, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testy")
suite.AssertOpSpan(nilParents[1], "Delete", agent.BucketName(), memd.CmdDelete.Name(), 1, false, "testy")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Delete", 1, false, false)
}
func (suite *StandardTestSuite) TestBasicInsert() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Delete(DeleteOptions{
Key: []byte("testz"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *DeleteResult, err error) {
s.Continue()
}))
s.Wait(0)
s.PushOp(agent.Add(AddOptions{
Key: []byte("testz"),
Value: []byte("[]"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Add operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(2, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Delete", agent.BucketName(), memd.CmdDelete.Name(), 1, false, "testz")
suite.AssertOpSpan(nilParents[1], "Add", agent.BucketName(), memd.CmdAdd.Name(), 1, false, "testz")
}
}
suite.VerifyKVMetrics(suite.meter, "Delete", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Add", 1, false, false)
}
func (suite *StandardTestSuite) TestBasicSetGet() {
spec := suite.StartTest("kv/crud/SetGet")
agent := spec.Agent
s := suite.GetHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("test-doc"),
Value: []byte("{}"),
CollectionName: spec.Collection,
ScopeName: spec.Scope,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
s.PushOp(agent.Get(GetOptions{
Key: []byte("test-doc"),
CollectionName: spec.Collection,
ScopeName: spec.Scope,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
if !bytes.Equal([]byte("{}"), res.Value) {
s.Fatalf("Value did not match")
}
})
}))
s.Wait(0)
suite.EndTest(spec)
if suite.Assert().Contains(spec.Tracer.Spans, nil) {
nilParents := spec.Tracer.Spans[nil]
if suite.Assert().Equal(2, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "test-doc")
suite.AssertOpSpan(nilParents[1], "Get", agent.BucketName(), memd.CmdGet.Name(), 1, false, "test-doc")
}
}
suite.VerifyKVMetrics(spec.Meter, "Set", 1, false, false)
suite.VerifyKVMetrics(spec.Meter, "Get", 1, false, false)
}
func (suite *StandardTestSuite) TestBasicCounters() {
agent, s := suite.GetAgentAndHarness()
// Counters
s.PushOp(agent.Delete(DeleteOptions{
Key: []byte("testCounters"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *DeleteResult, err error) {
s.Continue()
}))
s.Wait(0)
s.PushOp(agent.Increment(CounterOptions{
Key: []byte("testCounters"),
Delta: 5,
Initial: 11,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *CounterResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Increment operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
if res.Value != 11 {
s.Fatalf("Increment did not operate properly")
}
})
}))
s.Wait(0)
s.PushOp(agent.Increment(CounterOptions{
Key: []byte("testCounters"),
Delta: 5,
Initial: 22,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *CounterResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Increment operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
if res.Value != 16 {
s.Fatalf("Increment did not operate properly")
}
})
}))
s.Wait(0)
s.PushOp(agent.Decrement(CounterOptions{
Key: []byte("testCounters"),
Delta: 3,
Initial: 65,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *CounterResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Increment operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
if res.Value != 13 {
s.Fatalf("Increment did not operate properly")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(4, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Delete", agent.BucketName(), memd.CmdDelete.Name(), 1, false, "testCounters")
suite.AssertOpSpan(nilParents[1], "Increment", agent.BucketName(), memd.CmdIncrement.Name(), 1, false, "testCounters")
suite.AssertOpSpan(nilParents[2], "Increment", agent.BucketName(), memd.CmdIncrement.Name(), 1, false, "testCounters")
suite.AssertOpSpan(nilParents[3], "Decrement", agent.BucketName(), memd.CmdDecrement.Name(), 1, false, "testCounters")
}
}
suite.VerifyKVMetrics(suite.meter, "Delete", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Increment", 2, false, false)
suite.VerifyKVMetrics(suite.meter, "Decrement", 1, false, false)
}
func (suite *StandardTestSuite) TestBasicAdjoins() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testAdjoins"),
Value: []byte("there"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Continue()
}))
s.Wait(0)
s.PushOp(agent.Append(AdjoinOptions{
Key: []byte("testAdjoins"),
Value: []byte(" Frank!"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *AdjoinResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Append operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
s.PushOp(agent.Prepend(AdjoinOptions{
Key: []byte("testAdjoins"),
Value: []byte("Hello "),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *AdjoinResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Prepend operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
s.PushOp(agent.Get(GetOptions{
Key: []byte("testAdjoins"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
if string(res.Value) != "Hello there Frank!" {
s.Fatalf("Adjoin operations did not behave")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(4, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testAdjoins")
suite.AssertOpSpan(nilParents[1], "Append", agent.BucketName(), memd.CmdAppend.Name(), 1, false, "testAdjoins")
suite.AssertOpSpan(nilParents[2], "Prepend", agent.BucketName(), memd.CmdPrepend.Name(), 1, false, "testAdjoins")
suite.AssertOpSpan(nilParents[3], "Get", agent.BucketName(), memd.CmdGet.Name(), 1, false, "testAdjoins")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Append", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Prepend", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Get", 1, false, false)
}
func (suite *StandardTestSuite) TestExpiry() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testExpiry"),
Value: []byte("{}"),
Expiry: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
})
}))
s.Wait(0)
suite.TimeTravel(3000 * time.Millisecond)
s.PushOp(agent.Get(GetOptions{
Key: []byte("testExpiry"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
RetryStrategy: NewBestEffortRetryStrategy(nil),
}, func(res *GetResult, err error) {
s.Wrap(func() {
if !errors.Is(err, ErrDocumentNotFound) {
s.Fatalf("Get should have returned document not found")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(2, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testExpiry")
suite.AssertOpSpan(nilParents[1], "Get", agent.BucketName(), memd.CmdGet.Name(), 1, false, "testExpiry")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Get", 1, false, false)
}
func (suite *StandardTestSuite) TestTouch() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testTouch"),
Value: []byte("{}"),
Expiry: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.Touch(TouchOptions{
Key: []byte("testTouch"),
Expiry: 3,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *TouchResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Touch operation failed: %v", err)
}
})
}))
s.Wait(0)
suite.TimeTravel(1500 * time.Millisecond)
s.PushOp(agent.Get(GetOptions{
Key: []byte("testTouch"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get should have been successful")
}
})
}))
s.Wait(0)
suite.TimeTravel(3500 * time.Millisecond)
s.PushOp(agent.Get(GetOptions{
Key: []byte("testTouch"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if !errors.Is(err, ErrDocumentNotFound) {
s.Fatalf("Get should have returned document not found")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(4, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testTouch")
suite.AssertOpSpan(nilParents[1], "Touch", agent.BucketName(), memd.CmdTouch.Name(), 1, false, "testTouch")
suite.AssertOpSpan(nilParents[2], "Get", agent.BucketName(), memd.CmdGet.Name(), 1, false, "testTouch")
suite.AssertOpSpan(nilParents[3], "Get", agent.BucketName(), memd.CmdGet.Name(), 1, false, "testTouch")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Touch", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Get", 2, false, false)
}
func (suite *StandardTestSuite) TestGetAndTouch() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testGetAndTouch"),
Value: []byte("{}"),
Expiry: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.GetAndTouch(GetAndTouchOptions{
Key: []byte("testGetAndTouch"),
Expiry: 3,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetAndTouchResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Touch operation failed: %v", err)
}
})
}))
s.Wait(0)
suite.TimeTravel(1500 * time.Millisecond)
s.PushOp(agent.Get(GetOptions{
Key: []byte("testGetAndTouch"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get should have been successful")
}
})
}))
s.Wait(0)
suite.TimeTravel(3000 * time.Millisecond)
s.PushOp(agent.Get(GetOptions{
Key: []byte("testGetAndTouch"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if !errors.Is(err, ErrDocumentNotFound) {
s.Fatalf("Get should have returned document not found: %v", err)
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(4, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testGetAndTouch")
suite.AssertOpSpan(nilParents[1], "GetAndTouch", agent.BucketName(), memd.CmdGAT.Name(), 1, false, "testGetAndTouch")
suite.AssertOpSpan(nilParents[2], "Get", agent.BucketName(), memd.CmdGet.Name(), 1, false, "testGetAndTouch")
suite.AssertOpSpan(nilParents[3], "Get", agent.BucketName(), memd.CmdGet.Name(), 1, false, "testGetAndTouch")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "GetAndTouch", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Get", 2, false, false)
}
// This test will lock the document for 1 second, it will then perform set requests for up to 2 seconds,
// the operation should succeed within the 2 seconds.
func (suite *StandardTestSuite) TestRetrySet() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testRetrySet"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.GetAndLock(GetAndLockOptions{
Key: []byte("testRetrySet"),
LockTime: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetAndLockResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("GetAndLock operation failed: %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.Set(SetOptions{
Key: []byte("testRetrySet"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
RetryStrategy: NewBestEffortRetryStrategy(nil),
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(3, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testRetrySet")
suite.AssertOpSpan(nilParents[1], "GetAndLock", agent.BucketName(), memd.CmdGetLocked.Name(), 1, false, "testRetrySet")
suite.AssertOpSpan(nilParents[2], "Set", agent.BucketName(), memd.CmdGet.Name(), 1, true, "testRetrySet")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 2, false, false)
suite.VerifyKVMetrics(suite.meter, "GetAndLock", 1, false, false)
}
func (suite *StandardTestSuite) TestObserve() {
suite.EnsureSupportsFeature(TestFeatureReplicas)
agent, s := suite.GetAgentAndHarness()
if agent.HasCollectionsSupport() {
suite.T().Skip("Skipping test as observe does not support collections")
}
s.PushOp(agent.Set(SetOptions{
Key: []byte("testObserve"),
Value: []byte("there"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Continue()
}))
s.Wait(0)
s.PushOp(agent.Observe(ObserveOptions{
Key: []byte("testObserve"),
ReplicaIdx: 1,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *ObserveResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Observe operation failed: %v", err)
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(2, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testObserve")
suite.AssertOpSpan(nilParents[1], "Observe", agent.BucketName(), memd.CmdObserve.Name(), 1, false, "")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Observe", 1, false, false)
}
func (suite *StandardTestSuite) TestObserveSeqNo() {
suite.EnsureSupportsFeature(TestFeatureReplicas)
agent, s := suite.GetAgentAndHarness()
origMt := MutationToken{}
s.PushOp(agent.Set(SetOptions{
Key: []byte("testObserve"),
Value: []byte("there"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Initial set operation failed: %v", err)
}
mt := res.MutationToken
if mt.VbUUID == 0 && mt.SeqNo == 0 {
s.Skipf("ObserveSeqNo not supported by server")
}
origMt = mt
})
}))
s.Wait(0)
origCurSeqNo := SeqNo(0)
vbID, err := agent.kvMux.KeyToVbucket([]byte("testObserve"))
if err != nil {
s.Fatalf("KeyToVbucket operation failed: %v", err)
}
s.PushOp(agent.ObserveVb(ObserveVbOptions{
VbID: vbID,
VbUUID: origMt.VbUUID,
ReplicaIdx: 1,
}, func(res *ObserveVbResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("ObserveSeqNo operation failed: %v", err)
}
origCurSeqNo = res.CurrentSeqNo
})
}))
s.Wait(0)
newMt := MutationToken{}
s.PushOp(agent.Set(SetOptions{
Key: []byte("testObserve"),
Value: []byte("there"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Second set operation failed: %v", err)
}
newMt = res.MutationToken
})
}))
s.Wait(0)
vbID, err = agent.kvMux.KeyToVbucket([]byte("testObserve"))
if err != nil {
s.Fatalf("KeyToVbucket operation failed: %v", err)
}
s.PushOp(agent.ObserveVb(ObserveVbOptions{
VbID: vbID,
VbUUID: newMt.VbUUID,
ReplicaIdx: 1,
}, func(res *ObserveVbResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("ObserveSeqNo operation failed: %v", err)
}
if res.CurrentSeqNo < origCurSeqNo {
s.Fatalf("SeqNo does not appear to be working")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(4, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testObserve")
suite.AssertOpSpan(nilParents[1], "ObserveVb", agent.BucketName(), memd.CmdObserveSeqNo.Name(), 1, false, "")
suite.AssertOpSpan(nilParents[2], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testObserve")
suite.AssertOpSpan(nilParents[3], "ObserveVb", agent.BucketName(), memd.CmdObserveSeqNo.Name(), 1, false, "")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 2, false, false)
suite.VerifyKVMetrics(suite.meter, "ObserveVb", 2, false, false)
}
func (suite *StandardTestSuite) TestRandomGet() {
agent, s := suite.GetAgentAndHarness()
distkeys, err := MakeDistKeys(agent, time.Now().Add(2*time.Second))
suite.Require().Nil(err, err)
for _, k := range distkeys {
s.PushOp(agent.Set(SetOptions{
Key: []byte(k),
Value: []byte("Hello World!"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Couldn't store some items: %v", err)
}
})
}))
s.Wait(0)
}
s.PushOp(agent.GetRandom(GetRandomOptions{
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetRandomResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
if len(res.Key) == 0 {
s.Fatalf("Invalid key returned")
}
if len(res.Value) == 0 {
s.Fatalf("No value returned")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(len(distkeys)+1, len(nilParents)) {
for i, k := range distkeys {
suite.AssertOpSpan(nilParents[i], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, k)
}
suite.AssertOpSpan(nilParents[len(distkeys)], "GetRandom", agent.BucketName(), memd.CmdGetRandom.Name(), 1, false, "")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", len(distkeys), false, false)
suite.VerifyKVMetrics(suite.meter, "GetRandom", 1, false, false)
}
func (suite *StandardTestSuite) TestStats() {
agent, s := suite.GetAgentAndHarness()
snapshot, err := agent.ConfigSnapshot()
if err != nil {
suite.T().Fatalf("Failed to get config snapshot: %s", err)
}
numServers, err := snapshot.NumServers()
if err != nil {
suite.T().Fatalf("Failed to get num servers: %s", err)
}
s.PushOp(agent.Stats(StatsOptions{
Key: "",
}, func(res *StatsResult, err error) {
s.Wrap(func() {
if len(res.Servers) != numServers {
s.Fatalf("Didn't Get all stats!")
}
for srv, curStats := range res.Servers {
if curStats.Error != nil {
s.Fatalf("Got error %v in stats for %s", curStats.Error, srv)
}
if curStats.Stats == nil || len(curStats.Stats) == 0 {
s.Fatalf("Got no stats in stats for %s", srv)
}
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(1, len(nilParents)) {
p := agent.kvMux.NumPipelines()
suite.AssertTopLevelSpan(nilParents[0], "Stats", agent.BucketName())
spans := nilParents[0].Spans[memd.CmdStat.Name()]
if suite.Assert().Equal(p, len(spans)) {
for i := 0; i < len(spans); i++ {
span := spans[i]
suite.Assert().Equal(memd.CmdStat.Name(), span.Name)
suite.Assert().True(span.Finished)
netSpans := span.Spans[spanNameDispatchToServer]
if suite.Assert().Equal(1, len(netSpans)) {
suite.Assert().Equal(spanNameDispatchToServer, netSpans[0].Name)
suite.Assert().True(netSpans[0].Finished)
}
}
}
}
}
}
func (suite *StandardTestSuite) TestGetHttpEps() {
agent, _ := suite.GetAgentAndHarness()
// Relies on a 3.0.0+ server
n1qlEpList := agent.N1qlEps()
if len(n1qlEpList) == 0 {
suite.T().Fatalf("Failed to retrieve N1QL endpoint list")
}
mgmtEpList := agent.MgmtEps()
if len(mgmtEpList) == 0 {
suite.T().Fatalf("Failed to retrieve N1QL endpoint list")
}
capiEpList := agent.CapiEps()
if len(capiEpList) == 0 {
suite.T().Fatalf("Failed to retrieve N1QL endpoint list")
}
}
func (suite *StandardTestSuite) TestMemcachedBucket() {
suite.EnsureSupportsFeature(TestFeatureMemd)
spec := suite.StartTest(TestNameMemcachedBasic)
defer suite.EndTest(spec)
s := suite.GetHarness()
agent := spec.Agent
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitUntilReady failed with error: %v", err)
}
})
}))
s.Wait(6)
s.PushOp(agent.Set(SetOptions{
Key: []byte("test-doc"),
Value: []byte("value"),
CollectionName: spec.Collection,
ScopeName: spec.Scope,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Got error for Set: %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.Get(GetOptions{
Key: []byte("test-doc"),
CollectionName: spec.Collection,
ScopeName: spec.Scope,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Couldn't Get back key: %v", err)
}
if string(res.Value) != "value" {
s.Fatalf("Got back wrong value!")
}
})
}))
s.Wait(0)
// Try to perform Observe: should fail since this isn't supported on Memcached buckets
_, err := agent.Observe(ObserveOptions{
Key: []byte("key"),
CollectionName: spec.Collection,
ScopeName: spec.Scope,
}, func(res *ObserveResult, err error) {
s.Wrap(func() {
s.Fatalf("Scheduling should fail on memcached buckets!")
})
})
if !errors.Is(err, ErrFeatureNotAvailable) {
suite.T().Fatalf("Expected observe error for memcached bucket!")
}
if suite.Assert().Contains(spec.Tracer.Spans, nil) {
nilParents := spec.Tracer.Spans[nil]
if suite.Assert().Equal(3, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "key")
suite.AssertOpSpan(nilParents[1], "Get", agent.BucketName(), memd.CmdGet.Name(), 1, false, "key")
suite.AssertOpSpan(nilParents[2], "Observe", agent.BucketName(), memd.CmdObserve.Name(), 0, false, "")
}
}
suite.VerifyKVMetrics(spec.Meter, "Set", 1, false, false)
suite.VerifyKVMetrics(spec.Meter, "Get", 1, false, false)
suite.VerifyKVMetrics(spec.Meter, "Observe", 1, false, true)
}
func (suite *StandardTestSuite) TestFlagsRoundTrip() {
// Ensure flags are round-tripped with the server correctly.
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("flagskey"),
Value: []byte("{}"),
Flags: 0x99889988,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Got error for Set: %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.Get(GetOptions{
Key: []byte("flagskey"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Couldn't Get back key: %v", err)
}
if res.Flags != 0x99889988 {
s.Fatalf("flags failed to round-trip")
}
})
}))
s.Wait(0)
}
func (suite *StandardTestSuite) TestMetaOps() {
suite.EnsureSupportsFeature(TestFeatureGetMeta)
agent, s := suite.GetAgentAndHarness()
var currentCas Cas
// Set
s.PushOp(agent.Set(SetOptions{
Key: []byte("test"),
Value: []byte("{}"),
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed")
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
currentCas = res.Cas
})
}))
s.Wait(0)
// GetMeta
s.PushOp(agent.GetMeta(GetMetaOptions{
Key: []byte("test"),
}, func(res *GetMetaResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("GetMeta operation failed")
}
if res.Expiry != 0 {
s.Fatalf("Invalid expiry received")
}
if res.Deleted != 0 {
s.Fatalf("Invalid deleted flag received")
}
if res.Cas != currentCas {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(2, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "test")
suite.AssertOpSpan(nilParents[1], "GetMeta", agent.BucketName(), memd.CmdGetMeta.Name(), 1, false, "test")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "GetMeta", 1, false, false)
}
func (suite *StandardTestSuite) TestPing() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Ping(PingOptions{}, func(res *PingResult, err error) {
s.Wrap(func() {
if len(res.Services) == 0 {
s.Fatalf("Ping report contained no results")
}
})
}))
s.Wait(5)
}
func (suite *StandardTestSuite) TestDiagnostics() {
agent, _ := suite.GetAgentAndHarness()
report, err := agent.Diagnostics(DiagnosticsOptions{})
if err != nil {
suite.T().Fatalf("Failed to fetch diagnostics: %s", err)
}
if len(report.MemdConns) == 0 {
suite.T().Fatalf("Diagnostics report contained no results")
}
for _, conn := range report.MemdConns {
if conn.RemoteAddr == "" {
suite.T().Fatalf("Diagnostic report contained invalid entry")
}
}
}
type testAlternateAddressesRouteConfigMgr struct {
cfg *routeConfig
cfgCalled bool
}
func (taa *testAlternateAddressesRouteConfigMgr) OnNewRouteConfig(cfg *routeConfig) {
taa.cfgCalled = true
taa.cfg = cfg
}
func (suite *StandardTestSuite) TestAlternateAddressesEmptyStringConfig() {
cfgBk := suite.LoadConfigFromFile("testdata/bucket_config_with_external_addresses.json")
mgr := &testAlternateAddressesRouteConfigMgr{}
cfgManager := newConfigManager(configManagerProperties{
SrcMemdAddrs: []routeEndpoint{{Address: "192.168.132.234:32799"}},
UseTLS: false,
})
cfgManager.AddConfigWatcher(mgr)
cfgManager.OnNewConfig(cfgBk)
networkType := cfgManager.NetworkType()
if networkType != "external" {
suite.T().Fatalf("Expected agent networkType to be external, was %s", networkType)
}
for i, server := range mgr.cfg.kvServerList.NonSSLEndpoints {
cfgBkNode := cfgBk.NodesExt[i]
port := cfgBkNode.AltAddresses["external"].Ports.Kv
cfgBkServer := fmt.Sprintf("couchbase://%s:%d", cfgBkNode.AltAddresses["external"].Hostname, port)
if server.Address != cfgBkServer {
suite.T().Fatalf("Expected kv server to be %s but was %s", cfgBkServer, server.Address)
}
}
}
func (suite *StandardTestSuite) TestAlternateAddressesAutoConfig() {
cfgBk := suite.LoadConfigFromFile("testdata/bucket_config_with_external_addresses.json")
mgr := &testAlternateAddressesRouteConfigMgr{}
cfgManager := newConfigManager(configManagerProperties{
NetworkType: "auto",
SrcMemdAddrs: []routeEndpoint{{Address: "192.168.132.234:32799"}},
UseTLS: false,
})
cfgManager.AddConfigWatcher(mgr)
cfgManager.OnNewConfig(cfgBk)
networkType := cfgManager.NetworkType()
if networkType != "external" {
suite.T().Fatalf("Expected agent networkType to be external, was %s", networkType)
}
for i, server := range mgr.cfg.kvServerList.NonSSLEndpoints {
cfgBkNode := cfgBk.NodesExt[i]
port := cfgBkNode.AltAddresses["external"].Ports.Kv
cfgBkServer := fmt.Sprintf("couchbase://%s:%d", cfgBkNode.AltAddresses["external"].Hostname, port)
if server.Address != cfgBkServer {
suite.T().Fatalf("Expected kv server to be %s but was %s", cfgBkServer, server.Address)
}
}
}
func (suite *StandardTestSuite) TestAlternateAddressesAutoInternalConfig() {
cfgBk := suite.LoadConfigFromFile("testdata/bucket_config_with_external_addresses.json")
mgr := &testAlternateAddressesRouteConfigMgr{}
cfgManager := newConfigManager(configManagerProperties{
NetworkType: "auto",
SrcMemdAddrs: []routeEndpoint{{Address: "172.17.0.4:11210"}},
UseTLS: false,
})
cfgManager.AddConfigWatcher(mgr)
cfgManager.OnNewConfig(cfgBk)
networkType := cfgManager.NetworkType()
if networkType != "default" {
suite.T().Fatalf("Expected agent networkType to be default, was %s", networkType)
}
for i, server := range mgr.cfg.kvServerList.NonSSLEndpoints {
cfgBkNode := cfgBk.NodesExt[i]
port := cfgBkNode.Services.Kv
cfgBkServer := fmt.Sprintf("couchbase://%s:%d", cfgBkNode.Hostname, port)
if server.Address != cfgBkServer {
suite.T().Fatalf("Expected kv server to be %s but was %s", cfgBkServer, server.Address)
}
}
}
func (suite *StandardTestSuite) TestAlternateAddressesDefaultConfig() {
cfgBk := suite.LoadConfigFromFile("testdata/bucket_config_with_external_addresses.json")
mgr := &testAlternateAddressesRouteConfigMgr{}
cfgManager := newConfigManager(configManagerProperties{
NetworkType: "default",
SrcMemdAddrs: []routeEndpoint{{Address: "192.168.132.234:32799"}},
UseTLS: false,
})
cfgManager.AddConfigWatcher(mgr)
cfgManager.OnNewConfig(cfgBk)
networkType := cfgManager.NetworkType()
if networkType != "default" {
suite.T().Fatalf("Expected agent networkType to be default, was %s", networkType)
}
for i, server := range mgr.cfg.kvServerList.NonSSLEndpoints {
cfgBkNode := cfgBk.NodesExt[i]
port := cfgBkNode.Services.Kv
cfgBkServer := fmt.Sprintf("couchbase://%s:%d", cfgBkNode.Hostname, port)
if server.Address != cfgBkServer {
suite.T().Fatalf("Expected kv server to be %s but was %s", cfgBkServer, server.Address)
}
}
}
func (suite *StandardTestSuite) TestAlternateAddressesExternalConfig() {
cfgBk := suite.LoadConfigFromFile("testdata/bucket_config_with_external_addresses.json")
mgr := &testAlternateAddressesRouteConfigMgr{}
cfgManager := newConfigManager(configManagerProperties{
NetworkType: "external",
SrcMemdAddrs: []routeEndpoint{{Address: "192.168.132.234:32799"}},
UseTLS: false,
})
cfgManager.AddConfigWatcher(mgr)
cfgManager.OnNewConfig(cfgBk)
networkType := cfgManager.NetworkType()
if networkType != "external" {
suite.T().Fatalf("Expected agent networkType to be external, was %s", networkType)
}
for i, server := range mgr.cfg.kvServerList.NonSSLEndpoints {
cfgBkNode := cfgBk.NodesExt[i]
port := cfgBkNode.AltAddresses["external"].Ports.Kv
cfgBkServer := fmt.Sprintf("couchbase://%s:%d", cfgBkNode.AltAddresses["external"].Hostname, port)
if server.Address != cfgBkServer {
suite.T().Fatalf("Expected kv server to be %s but was %s", cfgBkServer, server.Address)
}
}
}
func (suite *StandardTestSuite) TestAlternateAddressesExternalConfigNoPorts() {
cfgBk := suite.LoadConfigFromFile("testdata/bucket_config_with_external_addresses_without_ports.json")
mgr := &testAlternateAddressesRouteConfigMgr{}
cfgManager := newConfigManager(configManagerProperties{
NetworkType: "external",
SrcMemdAddrs: []routeEndpoint{{Address: "192.168.132.234:32799"}},
UseTLS: false,
})
cfgManager.AddConfigWatcher(mgr)
cfgManager.OnNewConfig(cfgBk)
networkType := cfgManager.NetworkType()
if networkType != "external" {
suite.T().Fatalf("Expected agent networkType to be external, was %s", networkType)
}
for i, server := range mgr.cfg.kvServerList.NonSSLEndpoints {
cfgBkNode := cfgBk.NodesExt[i]
port := cfgBkNode.Services.Kv
cfgBkServer := fmt.Sprintf("couchbase://%s:%d", cfgBkNode.AltAddresses["external"].Hostname, port)
if server.Address != cfgBkServer {
suite.T().Fatalf("Expected kv server to be %s but was %s", cfgBkServer, server.Address)
}
}
}
func (suite *StandardTestSuite) TestAlternateAddressesInvalidConfig() {
cfgBk := suite.LoadConfigFromFile("testdata/bucket_config_with_external_addresses.json")
mgr := &testAlternateAddressesRouteConfigMgr{}
cfgManager := newConfigManager(configManagerProperties{
NetworkType: "invalid",
SrcMemdAddrs: []routeEndpoint{{Address: "192.168.132.234:32799"}},
UseTLS: false,
})
cfgManager.AddConfigWatcher(mgr)
cfgManager.OnNewConfig(cfgBk)
networkType := cfgManager.NetworkType()
if networkType != "invalid" {
suite.T().Fatalf("Expected agent networkType to be invalid, was %s", networkType)
}
if mgr.cfgCalled {
suite.T().Fatalf("Expected route config to not be propagated, was propagated")
}
}
func (suite *StandardTestSuite) TestAgentWaitForConfigSnapshot() {
cfg := makeAgentConfig(globalTestConfig)
cfg.BucketName = globalTestConfig.BucketName
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
var snapshot *ConfigSnapshot
s.PushOp(agent.WaitForConfigSnapshot(time.Now().Add(5*time.Second), WaitForConfigSnapshotOptions{}, func(result *WaitForConfigSnapshotResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitForConfigSnapshot failed with error: %v", err)
}
snapshot = result.Snapshot
})
}))
s.Wait(6)
suite.Assert().True(snapshot.RevID() > -1)
}
func (suite *StandardTestSuite) TestAgentWaitForConfigSnapshotSteadyState() {
cfg := makeAgentConfig(globalTestConfig)
cfg.BucketName = globalTestConfig.BucketName
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
var snapshot *ConfigSnapshot
s.PushOp(agent.WaitForConfigSnapshot(time.Now().Add(5*time.Second), WaitForConfigSnapshotOptions{}, func(result *WaitForConfigSnapshotResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitForConfigSnapshot failed with error: %v", err)
}
snapshot = result.Snapshot
})
}))
s.Wait(6)
suite.Assert().True(snapshot.RevID() > -1)
// Run it again, we know that the agent has already seen a config by now.
s.PushOp(agent.WaitForConfigSnapshot(time.Now().Add(5*time.Second), WaitForConfigSnapshotOptions{}, func(result *WaitForConfigSnapshotResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitForConfigSnapshot failed with error: %v", err)
}
snapshot = result.Snapshot
})
}))
s.Wait(6)
suite.Assert().True(snapshot.RevID() > -1)
}
func (suite *StandardTestSuite) TestAgentWaitUntilReadyGCCCP() {
suite.EnsureSupportsFeature(TestFeatureGCCCP)
cfg := makeAgentConfig(globalTestConfig)
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitUntilReady failed with error: %v", err)
}
})
}))
s.Wait(6)
s.PushOp(agent.Ping(PingOptions{
ServiceTypes: []ServiceType{N1qlService},
N1QLDeadline: time.Now().Add(5 * time.Second),
}, func(result *PingResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Ping failed with error: %v", err)
}
})
}))
s.Wait(0)
}
func (suite *StandardTestSuite) VerifyConnectedToBucket(agent *Agent, s *TestSubHarness, test, collection, scope string) {
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitUntilReady failed with error: %v", err)
}
})
}))
s.Wait(6)
s.PushOp(agent.Set(SetOptions{
Key: []byte(test),
Value: []byte("{}"),
CollectionName: collection,
ScopeName: scope,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Got error for Set: %v", err)
}
})
}))
s.Wait(0)
}
func (suite *StandardTestSuite) VerifyConnectedToBucketHTTP(agent *Agent, bucket string, s *TestSubHarness, test string) {
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitUntilReady failed with error: %v", err)
}
})
}))
s.Wait(6)
req := &HTTPRequest{
Service: MgmtService,
Path: fmt.Sprintf("/pools/default/buckets/%s", bucket),
Method: "GET",
Headers: make(map[string]string),
Deadline: time.Now().Add(2 * time.Second),
}
s.PushOp(agent.DoHTTPRequest(req, func(res *HTTPResponse, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Got error for HTTP request: %v", err)
}
res.Body.Close()
})
}))
s.Wait(0)
}
func (suite *StandardTestSuite) TestAgentWaitUntilReadyBucket() {
cfg := makeAgentConfig(globalTestConfig)
cfg.BucketName = globalTestConfig.BucketName
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestAgentWaitUntilReadyBucket", suite.CollectionName, suite.ScopeName)
}
func (suite *StandardTestSuite) TestAgentGroupWaitUntilReadyGCCCP() {
suite.EnsureSupportsFeature(TestFeatureGCCCP)
cfg := suite.makeAgentGroupConfig(globalTestConfig)
ag, err := CreateAgentGroup(&cfg)
suite.Require().Nil(err, err)
defer ag.Close()
s := suite.GetHarness()
s.PushOp(ag.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitUntilReady failed with error: %v", err)
}
})
}))
s.Wait(6)
s.PushOp(ag.Ping(PingOptions{
ServiceTypes: []ServiceType{N1qlService},
N1QLDeadline: time.Now().Add(5 * time.Second),
}, func(result *PingResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Ping failed with error: %v", err)
}
})
}))
s.Wait(0)
}
// This test cannot run against mock as the mock does not respond with 200 status code for all of the endpoints.
func (suite *StandardTestSuite) TestAgentGroupWaitUntilReadyBucket() {
cfg := suite.makeAgentGroupConfig(globalTestConfig)
ag, err := CreateAgentGroup(&cfg)
suite.Require().Nil(err, err)
defer ag.Close()
s := suite.GetHarness()
err = ag.OpenBucket(globalTestConfig.BucketName)
suite.Require().Nil(err, err)
agent := ag.GetAgent("default")
suite.Require().NotNil(agent)
suite.VerifyConnectedToBucket(agent, s, "TestAgentGroupWaitUntilReadyBucket", suite.CollectionName, suite.ScopeName)
}
func (suite *StandardTestSuite) TestConnectHTTPOnlyDefaultPort() {
cfg := makeAgentConfig(globalTestConfig)
if len(cfg.SeedConfig.HTTPAddrs) == 0 {
suite.T().Skip("Skipping test due to no HTTP addresses")
}
addr1 := cfg.SeedConfig.HTTPAddrs[0]
port := strings.Split(addr1, ":")[1]
if port != "8091" {
suite.T().Skipf("Skipping test due to non default port %s", port)
}
cfg.SeedConfig.HTTPAddrs = []string{addr1}
cfg.SeedConfig.MemdAddrs = []string{}
cfg.BucketName = globalTestConfig.BucketName
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestConnectHTTPOnlyDefaultPort", suite.CollectionName, suite.ScopeName)
}
func (suite *StandardTestSuite) TestConnectHTTPOnlyDefaultPortSSL() {
suite.EnsureSupportsFeature(TestFeatureSsl)
cfg := makeAgentConfig(globalTestConfig)
if len(cfg.SeedConfig.HTTPAddrs) == 0 {
suite.T().Skip("Skipping test due to no HTTP addresses")
}
addr1 := cfg.SeedConfig.HTTPAddrs[0]
parts := strings.Split(addr1, ":")
if parts[1] != "8091" {
suite.T().Skipf("Skipping test due to non default port %s", parts[1])
}
cfg.SeedConfig.HTTPAddrs = []string{parts[0] + ":" + "18091"}
cfg.SeedConfig.MemdAddrs = []string{}
cfg.SecurityConfig.UseTLS = true
// SkipVerify
cfg.SecurityConfig.TLSRootCAProvider = func() *x509.CertPool {
return nil
}
cfg.BucketName = globalTestConfig.BucketName
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestConnectHTTPOnlyDefaultPortSSL", suite.CollectionName, suite.ScopeName)
}
func (suite *StandardTestSuite) TestConnectHTTPOnlyDefaultPortFastFailInvalidBucket() {
cfg := makeAgentConfig(globalTestConfig)
if len(cfg.SeedConfig.HTTPAddrs) == 0 {
suite.T().Skip("Skipping test due to no HTTP addresses")
}
// This test purposefully triggers error cases.
globalTestLogger.SuppressWarnings(true)
defer globalTestLogger.SuppressWarnings(false)
addr1 := cfg.SeedConfig.HTTPAddrs[0]
port := strings.Split(addr1, ":")[1]
if port != "8091" {
suite.T().Skipf("Skipping test due to non default port %s", port)
}
cfg.SeedConfig.HTTPAddrs = []string{addr1}
cfg.SeedConfig.MemdAddrs = []string{}
cfg.BucketName = "idontexist"
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
start := time.Now()
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{
RetryStrategy: newFailFastRetryStrategy(),
}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err == nil {
s.Fatalf("WaitUntilReady failed without error")
}
if !errors.Is(err, ErrAuthenticationFailure) {
s.Fatalf("WaitUntilReady should have failed with auth error but was %v", err)
}
if time.Since(start) > 5*time.Second {
s.Fatalf("WaitUntilReady should have failed before the timeout duration, was %s", time.Since(start))
}
})
}))
s.Wait(6)
}
func (suite *StandardTestSuite) TestConnectHTTPOnlyNonDefaultPort() {
cfg := makeAgentConfig(globalTestConfig)
if len(cfg.SeedConfig.HTTPAddrs) == 0 {
suite.T().Skip("Skipping test due to no HTTP addresses")
}
addr1 := cfg.SeedConfig.HTTPAddrs[0]
port := strings.Split(addr1, ":")[1]
if port == "8091" {
suite.T().Skipf("Skipping test due to default port %s", port)
}
cfg.SeedConfig.HTTPAddrs = []string{addr1}
cfg.SeedConfig.MemdAddrs = []string{}
cfg.BucketName = globalTestConfig.BucketName
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestConnectHTTPOnlyNonDefaultPort", suite.CollectionName, suite.ScopeName)
}
func (suite *StandardTestSuite) TestConnectHTTPOnlyNonDefaultPortNoBucket() {
cfg := makeAgentConfig(globalTestConfig)
if len(cfg.SeedConfig.HTTPAddrs) == 0 {
suite.T().Skip("Skipping test due to no HTTP addresses")
}
addr1 := cfg.SeedConfig.HTTPAddrs[0]
port := strings.Split(addr1, ":")[1]
if port == "8091" {
suite.T().Skipf("Skipping test due to default port %s", port)
}
cfg.SeedConfig.HTTPAddrs = []string{addr1}
cfg.SeedConfig.MemdAddrs = []string{}
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if !errors.Is(err, ErrTimeout) {
s.Fatalf("WaitUntilReady should have timed out but didn't with error: %v", err)
}
})
}))
s.Wait(6)
}
func (suite *StandardTestSuite) TestConnectHTTPOnlyNonDefaultPortFastFailInvalidBucket() {
cfg := makeAgentConfig(globalTestConfig)
if len(cfg.SeedConfig.HTTPAddrs) == 0 {
suite.T().Skip("Skipping test due to no HTTP addresses")
}
// This test purposefully triggers error cases.
globalTestLogger.SuppressWarnings(true)
defer globalTestLogger.SuppressWarnings(false)
addr1 := cfg.SeedConfig.HTTPAddrs[0]
port := strings.Split(addr1, ":")[1]
if port == "8091" {
suite.T().Skipf("Skipping test due to default port %s", port)
}
cfg.SeedConfig.HTTPAddrs = []string{addr1}
cfg.SeedConfig.MemdAddrs = []string{}
cfg.BucketName = "idontexist"
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
start := time.Now()
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{
RetryStrategy: newFailFastRetryStrategy(),
}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err == nil {
s.Fatalf("WaitUntilReady failed without error")
}
if !errors.Is(err, ErrAuthenticationFailure) {
s.Fatalf("WaitUntilReady should have failed with auth error but was %v", err)
}
if time.Since(start) > 5*time.Second {
s.Fatalf("WaitUntilReady should have failed before the timeout duration, was %s", time.Since(start))
}
})
}))
s.Wait(6)
}
func (suite *StandardTestSuite) TestConnectMemdOnlyDefaultPort() {
cfg := makeAgentConfig(globalTestConfig)
if len(cfg.SeedConfig.MemdAddrs) == 0 {
suite.T().Skip("Skipping test due to no Memd addresses")
}
addr1 := cfg.SeedConfig.MemdAddrs[0]
port := strings.Split(addr1, ":")[1]
if port != "11210" {
suite.T().Skipf("Skipping test due to non default port %s", port)
}
cfg.SeedConfig.HTTPAddrs = []string{}
cfg.SeedConfig.MemdAddrs = []string{addr1}
cfg.BucketName = globalTestConfig.BucketName
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestConnectMemdOnlyDefaultPort", suite.CollectionName, suite.ScopeName)
}
func (suite *StandardTestSuite) TestConnectMemdOnlyDefaultPortSSL() {
suite.EnsureSupportsFeature(TestFeatureSsl)
cfg := makeAgentConfig(globalTestConfig)
if len(cfg.SeedConfig.MemdAddrs) == 0 {
suite.T().Skip("Skipping test due to no memd addresses")
}
addr1 := cfg.SeedConfig.MemdAddrs[0]
parts := strings.Split(addr1, ":")
if parts[1] != "11210" {
suite.T().Skipf("Skipping test due to non default port %s", parts[1])
}
cfg.SeedConfig.HTTPAddrs = []string{}
cfg.SeedConfig.MemdAddrs = []string{parts[0] + ":11207"}
cfg.SecurityConfig.UseTLS = true
// SkipVerify
cfg.SecurityConfig.TLSRootCAProvider = func() *x509.CertPool {
return nil
}
cfg.BucketName = globalTestConfig.BucketName
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestConnectMemdOnlyDefaultPortSSL", suite.CollectionName, suite.ScopeName)
}
func (suite *StandardTestSuite) TestConnectMemdOnlyNonDefaultPort() {
cfg := makeAgentConfig(globalTestConfig)
if len(cfg.SeedConfig.MemdAddrs) == 0 {
suite.T().Skip("Skipping test due to no memd addresses")
}
addr1 := cfg.SeedConfig.MemdAddrs[0]
port := strings.Split(addr1, ":")[1]
if port == "8091" {
suite.T().Skipf("Skipping test due to default port %s", port)
}
cfg.SeedConfig.HTTPAddrs = []string{}
cfg.SeedConfig.MemdAddrs = []string{addr1}
cfg.BucketName = globalTestConfig.BucketName
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestConnectMemdOnlyNonDefaultPort", suite.CollectionName, suite.ScopeName)
}
func (suite *StandardTestSuite) TestConnectMemdOnlyDefaultPortFastFailInvalidBucket() {
cfg := makeAgentConfig(globalTestConfig)
if len(cfg.SeedConfig.MemdAddrs) == 0 {
suite.T().Skip("Skipping test due to no memd addresses")
}
// This test purposefully triggers error cases.
globalTestLogger.SuppressWarnings(true)
defer globalTestLogger.SuppressWarnings(false)
addr1 := cfg.SeedConfig.MemdAddrs[0]
port := strings.Split(addr1, ":")[1]
if port != "11210" {
suite.T().Skipf("Skipping test due to non default port %s", port)
}
cfg.SeedConfig.HTTPAddrs = []string{}
cfg.SeedConfig.MemdAddrs = []string{addr1}
cfg.BucketName = "idontexist"
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
start := time.Now()
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{
RetryStrategy: newFailFastRetryStrategy(),
}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err == nil {
s.Fatalf("WaitUntilReady failed without error")
}
if !errors.Is(err, ErrAuthenticationFailure) {
s.Fatalf("WaitUntilReady should have failed with auth error but was %v", err)
}
if time.Since(start) > 5*time.Second {
s.Fatalf("WaitUntilReady should have failed before the timeout duration, was %s", time.Since(start))
}
})
}))
s.Wait(6)
}
// This test tests that given an address any connections to it will be made not using SSL whilst other connections will
// be made using TLS.
func (suite *StandardTestSuite) TestAgentNSServerScheme() {
suite.EnsureSupportsFeature(TestFeatureSsl)
defaultAgent := suite.DefaultAgent()
snapshot, err := defaultAgent.kvMux.PipelineSnapshot()
suite.Require().Nil(err, err)
if snapshot.NumPipelines() == 1 {
suite.T().Skip("Skipping test due to cluster only containing one node")
}
srcCfg := makeAgentConfig(globalTestConfig)
if len(srcCfg.SeedConfig.HTTPAddrs) == 0 {
suite.T().Skip("Skipping test due to no HTTP addresses")
}
seedAddr := srcCfg.SeedConfig.HTTPAddrs[0]
parts := strings.Split(seedAddr, ":")
if parts[1] != "8091" && parts[1] != "11210" {
// This should work with non default ports but it makes the test logic too complicated.
// This implicitly means that if TLS is enabled then this test won't run.
suite.T().Skip("Skipping test due to non default ports have been supplied")
}
connstr := fmt.Sprintf("ns_server://%s", seedAddr)
config := AgentConfig{}
err = config.FromConnStr(connstr)
suite.Require().Nil(err, err)
config.IoConfig = IoConfig{
UseDurations: true,
UseMutationTokens: true,
UseCollections: true,
UseOutOfOrderResponses: true,
}
config.SecurityConfig.Auth = globalTestConfig.Authenticator
config.SecurityConfig.UseTLS = true
config.SecurityConfig.TLSRootCAProvider = func() *x509.CertPool {
return nil
}
config.BucketName = globalTestConfig.BucketName
agent, err := CreateAgent(&config)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestAgentNSServerScheme", suite.CollectionName, suite.ScopeName)
kvMuxState := agent.kvMux.getState()
kvEps := kvMuxState.kvServerList
for _, ep := range kvEps {
epParts := strings.Split(ep.Address, "://")
hostport := strings.Split(epParts[1], ":")
if parts[0] == hostport[0] {
suite.Assert().Equal("couchbase", epParts[0])
suite.Assert().Equal("11210", hostport[1])
} else {
suite.Assert().Equal("couchbases", epParts[0])
suite.Assert().Equal("11207", hostport[1])
}
}
httpMuxState := agent.httpMux.Get()
mgmtEps := httpMuxState.mgmtEpList
for _, ep := range mgmtEps {
epParts := strings.Split(ep.Address, "://")
hostport := strings.Split(epParts[1], ":")
if hostport[0] == parts[0] {
suite.Assert().Equal("http", epParts[0])
suite.Assert().Equal(hostport[1], "8091")
} else {
suite.Assert().Equal("https", epParts[0])
suite.Assert().NotEqual(hostport[1], "8091")
}
}
}
// These functions are likely temporary.
type testManifestWithError struct {
Manifest Manifest
Err error
}
func testCreateScope(name, bucketName string, agent *Agent) (*Manifest, error) {
data := url.Values{}
data.Set("name", name)
req := &HTTPRequest{
Service: MgmtService,
Path: fmt.Sprintf("/pools/default/buckets/%s/scopes", bucketName),
Method: "POST",
Body: []byte(data.Encode()),
Headers: make(map[string]string),
Deadline: time.Now().Add(10 * time.Second),
}
req.Headers["Content-Type"] = "application/x-www-form-urlencoded"
resCh := make(chan *HTTPResponse)
errCh := make(chan error)
_, err := agent.DoHTTPRequest(req, func(response *HTTPResponse, err error) {
if err != nil {
errCh <- err
return
}
resCh <- response
})
if err != nil {
return nil, err
}
var resp *HTTPResponse
select {
case respErr := <-errCh:
if respErr != nil {
return nil, respErr
}
case res := <-resCh:
resp = res
}
if resp.StatusCode >= 300 {
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("could not create scope, status code: %d", resp.StatusCode)
}
err = resp.Body.Close()
if err != nil {
logDebugf("Failed to close response body")
}
return nil, fmt.Errorf("could not create scope, %s", string(data))
}
respBody := struct {
UID string `json:"uid"`
}{}
jsonDec := json.NewDecoder(resp.Body)
err = jsonDec.Decode(&respBody)
if err != nil {
return nil, err
}
err = resp.Body.Close()
if err != nil {
return nil, err
}
uid, err := strconv.ParseInt(respBody.UID, 16, 64)
if err != nil {
return nil, err
}
timer := time.NewTimer(20 * time.Second)
waitCh := make(chan testManifestWithError, 1)
go waitForManifest(agent, uint64(uid), waitCh)
for {
select {
case <-timer.C:
return nil, errors.New("wait time for scope to become available expired")
case manifest := <-waitCh:
if manifest.Err != nil {
return nil, manifest.Err
}
return &manifest.Manifest, nil
}
}
}
func testDeleteScope(name, bucketName string, agent *Agent, waitForDeletion bool) (*Manifest, error) {
data := url.Values{}
data.Set("name", name)
req := &HTTPRequest{
Service: MgmtService,
Path: fmt.Sprintf("/pools/default/buckets/%s/scopes/%s", bucketName, name),
Method: "DELETE",
Headers: make(map[string]string),
Deadline: time.Now().Add(10 * time.Second),
}
resCh := make(chan *HTTPResponse)
errCh := make(chan error)
_, err := agent.DoHTTPRequest(req, func(response *HTTPResponse, err error) {
if err != nil {
errCh <- err
return
}
resCh <- response
})
if err != nil {
return nil, err
}
var resp *HTTPResponse
select {
case respErr := <-errCh:
if respErr != nil {
return nil, respErr
}
case res := <-resCh:
resp = res
}
if err != nil {
return nil, err
}
if resp.StatusCode >= 300 {
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("could not delete scope, status code: %d", resp.StatusCode)
}
err = resp.Body.Close()
if err != nil {
logDebugf("Failed to close response body")
}
return nil, fmt.Errorf("could not delete scope, %s", string(data))
}
respBody := struct {
UID string `json:"uid"`
}{}
jsonDec := json.NewDecoder(resp.Body)
err = jsonDec.Decode(&respBody)
if err != nil {
return nil, err
}
err = resp.Body.Close()
if err != nil {
return nil, err
}
uid, err := strconv.ParseInt(respBody.UID, 16, 64)
if err != nil {
return nil, err
}
timer := time.NewTimer(20 * time.Second)
waitCh := make(chan testManifestWithError, 1)
go waitForManifest(agent, uint64(uid), waitCh)
for {
select {
case <-timer.C:
return nil, errors.New("wait time for scope to become deleted expired")
case manifest := <-waitCh:
if manifest.Err != nil {
return nil, manifest.Err
}
return &manifest.Manifest, nil
}
}
}
func testCreateCollection(name, scopeName, bucketName string, agent *Agent) (*Manifest, error) {
if scopeName == "" {
scopeName = "_default"
}
if name == "" {
name = "_default"
}
data := url.Values{}
data.Set("name", name)
req := &HTTPRequest{
Service: MgmtService,
Path: fmt.Sprintf("/pools/default/buckets/%s/scopes/%s/collections", bucketName, scopeName),
Method: "POST",
Body: []byte(data.Encode()),
Headers: make(map[string]string),
Deadline: time.Now().Add(10 * time.Second),
}
req.Headers["Content-Type"] = "application/x-www-form-urlencoded"
resCh := make(chan *HTTPResponse)
errCh := make(chan error)
_, err := agent.DoHTTPRequest(req, func(response *HTTPResponse, err error) {
if err != nil {
errCh <- err
return
}
resCh <- response
})
if err != nil {
return nil, err
}
var resp *HTTPResponse
select {
case respErr := <-errCh:
if respErr != nil {
return nil, respErr
}
case res := <-resCh:
resp = res
}
if resp.StatusCode >= 300 {
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("could not create collection, status code: %d", resp.StatusCode)
}
err = resp.Body.Close()
if err != nil {
logDebugf("Failed to close response body")
}
return nil, fmt.Errorf("could not create collection, %s", string(data))
}
respBody := struct {
UID string `json:"uid"`
}{}
jsonDec := json.NewDecoder(resp.Body)
err = jsonDec.Decode(&respBody)
if err != nil {
return nil, err
}
err = resp.Body.Close()
if err != nil {
return nil, err
}
uid, err := strconv.ParseInt(respBody.UID, 16, 64)
if err != nil {
return nil, err
}
timer := time.NewTimer(20 * time.Second)
waitCh := make(chan testManifestWithError, 1)
go waitForManifest(agent, uint64(uid), waitCh)
for {
select {
case <-timer.C:
return nil, errors.New("wait time for collection to become available expired")
case manifest := <-waitCh:
if manifest.Err != nil {
return nil, manifest.Err
}
return &manifest.Manifest, nil
}
}
}
func testDeleteCollection(name, scopeName, bucketName string, agent *Agent, waitForDeletion bool) (*Manifest, error) {
if scopeName == "" {
scopeName = "_default"
}
if name == "" {
name = "_default"
}
data := url.Values{}
data.Set("name", name)
req := &HTTPRequest{
Service: MgmtService,
Path: fmt.Sprintf("/pools/default/buckets/%s/scopes/%s/collections/%s", bucketName, scopeName, name),
Method: "DELETE",
Headers: make(map[string]string),
Deadline: time.Now().Add(10 * time.Second),
}
resCh := make(chan *HTTPResponse)
errCh := make(chan error)
_, err := agent.DoHTTPRequest(req, func(response *HTTPResponse, err error) {
if err != nil {
errCh <- err
return
}
resCh <- response
})
if err != nil {
return nil, err
}
var resp *HTTPResponse
select {
case respErr := <-errCh:
if respErr != nil {
return nil, respErr
}
case res := <-resCh:
resp = res
}
if err != nil {
return nil, err
}
if resp.StatusCode >= 300 {
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("could not delete collection, status code: %d", resp.StatusCode)
}
err = resp.Body.Close()
if err != nil {
logDebugf("Failed to close response body")
}
return nil, fmt.Errorf("could not delete collection, %s", string(data))
}
respBody := struct {
UID string `json:"uid"`
}{}
jsonDec := json.NewDecoder(resp.Body)
err = jsonDec.Decode(&respBody)
if err != nil {
return nil, err
}
err = resp.Body.Close()
if err != nil {
return nil, err
}
uid, err := strconv.ParseInt(respBody.UID, 16, 64)
if err != nil {
return nil, err
}
timer := time.NewTimer(20 * time.Second)
waitCh := make(chan testManifestWithError, 1)
go waitForManifest(agent, uint64(uid), waitCh)
for {
select {
case <-timer.C:
return nil, errors.New("wait time for collection to become deleted expired")
case manifest := <-waitCh:
if manifest.Err != nil {
return nil, manifest.Err
}
return &manifest.Manifest, nil
}
}
}
func waitForManifest(agent *Agent, manifestID uint64, manifestCh chan testManifestWithError) {
var manifest Manifest
for manifest.UID != manifestID {
setCh := make(chan struct{})
agent.GetCollectionManifest(GetCollectionManifestOptions{}, func(result *GetCollectionManifestResult, err error) {
if err != nil {
log.Println(err.Error())
close(setCh)
manifestCh <- testManifestWithError{Err: err}
return
}
err = json.Unmarshal(result.Manifest, &manifest)
if err != nil {
log.Println(err.Error())
close(setCh)
manifestCh <- testManifestWithError{Err: err}
return
}
if manifest.UID == manifestID {
close(setCh)
manifestCh <- testManifestWithError{Manifest: manifest}
return
}
setCh <- struct{}{}
})
<-setCh
time.Sleep(500 * time.Millisecond)
}
}
func dumpManifest(agent *Agent, t *testing.T) {
waitCh := make(chan struct{}, 1)
_, err := agent.GetCollectionManifest(GetCollectionManifestOptions{}, func(result *GetCollectionManifestResult, err error) {
if err != nil {
t.Logf("Failed to Get Collection Manifest: %v", err)
return
}
var manifest Manifest
err = json.Unmarshal(result.Manifest, &manifest)
if err != nil {
t.Logf("Failed to unmarshal manifest: %v", err)
return
}
t.Logf("Manifest: %+v", manifest)
waitCh <- struct{}{}
})
if err != nil {
t.Logf("Failed to send GetCollectionManifest: %v", err)
return
}
<-waitCh
}
gocbcore-10.2.3/agentgroup.go 0000664 0000000 0000000 00000017302 14417540156 0016074 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"sync"
"time"
)
// AgentGroup represents a collection of agents that can be used for performing operations
// against a cluster. It holds an internal special agent type which does not create its own
// memcached connections but registers itself for cluster config updates on all agents that
// are created through it.
type AgentGroup struct {
agentsLock sync.Mutex
boundAgents map[string]*Agent
// clusterAgent holds no memcached connections but can be used for cluster level (i.e. http) operations.
// It sets its own internal state by listening to cluster config updates on underlying agents.
clusterAgent *clusterAgent
config *AgentGroupConfig
}
// CreateAgentGroup will return a new AgentGroup with a base config of the config provided.
// Volatile: AgentGroup is subject to change or removal.
func CreateAgentGroup(config *AgentGroupConfig) (*AgentGroup, error) {
logInfof("SDK Version: gocbcore/%s", goCbCoreVersionStr)
logInfof("Creating new agent group: %+v", config)
c := config.toAgentConfig()
agent, err := CreateAgent(c)
if err != nil {
return nil, err
}
ag := &AgentGroup{
config: config,
boundAgents: make(map[string]*Agent),
}
ag.clusterAgent, err = createClusterAgent(&clusterAgentConfig{
UserAgent: config.UserAgent,
SeedConfig: config.SeedConfig,
SecurityConfig: config.SecurityConfig,
HTTPConfig: config.HTTPConfig,
TracerConfig: config.TracerConfig,
MeterConfig: config.MeterConfig,
DefaultRetryStrategy: config.DefaultRetryStrategy,
CircuitBreakerConfig: config.CircuitBreakerConfig,
})
if err != nil {
return nil, err
}
ag.clusterAgent.RegisterWith(agent.cfgManager, agent.dialer)
ag.boundAgents[config.BucketName] = agent
return ag, nil
}
// OpenBucket will attempt to open a new bucket against the cluster.
// If an agent using the specified bucket name already exists then this will not open a new connection.
func (ag *AgentGroup) OpenBucket(bucketName string) error {
if bucketName == "" {
return wrapError(errInvalidArgument, "bucket name cannot be empty")
}
existing := ag.GetAgent(bucketName)
if existing != nil {
return nil
}
config := ag.config.toAgentConfig()
config.BucketName = bucketName
agent, err := CreateAgent(config)
if err != nil {
return err
}
ag.clusterAgent.RegisterWith(agent.cfgManager, agent.dialer)
ag.agentsLock.Lock()
ag.boundAgents[bucketName] = agent
ag.maybeCloseGlobalAgent()
ag.agentsLock.Unlock()
return nil
}
// GetAgent will return the agent, if any, corresponding to the bucket name specified.
func (ag *AgentGroup) GetAgent(bucketName string) *Agent {
if bucketName == "" {
// We don't allow access to the global level agent. We close that agent on OpenBucket so we don't want
// to return an agent that we then later close. Doing so would only lead to pain.
return nil
}
ag.agentsLock.Lock()
existingAgent := ag.boundAgents[bucketName]
ag.agentsLock.Unlock()
if existingAgent != nil {
return existingAgent
}
return nil
}
// Close will close all underlying agents.
func (ag *AgentGroup) Close() error {
var firstError error
ag.agentsLock.Lock()
for _, agent := range ag.boundAgents {
ag.clusterAgent.UnregisterWith(agent.cfgManager, agent.dialer)
if err := agent.Close(); err != nil && firstError == nil {
firstError = err
}
}
ag.agentsLock.Unlock()
if err := ag.clusterAgent.Close(); err != nil && firstError == nil {
firstError = err
}
return firstError
}
// N1QLQuery executes a N1QL query against a random connected agent.
// If no agent is connected then this will block until one is available or the deadline is reached.
func (ag *AgentGroup) N1QLQuery(opts N1QLQueryOptions, cb N1QLQueryCallback) (PendingOp, error) {
return ag.clusterAgent.N1QLQuery(opts, cb)
}
// PreparedN1QLQuery executes a prepared N1QL query against a random connected agent.
// If no agent is connected then this will block until one is available or the deadline is reached.
func (ag *AgentGroup) PreparedN1QLQuery(opts N1QLQueryOptions, cb N1QLQueryCallback) (PendingOp, error) {
return ag.clusterAgent.PreparedN1QLQuery(opts, cb)
}
// AnalyticsQuery executes an analytics query against a random connected agent.
// If no agent is connected then this will block until one is available or the deadline is reached.
func (ag *AgentGroup) AnalyticsQuery(opts AnalyticsQueryOptions, cb AnalyticsQueryCallback) (PendingOp, error) {
return ag.clusterAgent.AnalyticsQuery(opts, cb)
}
// SearchQuery executes a Search query against a random connected agent.
// If no agent is connected then this will block until one is available or the deadline is reached.
func (ag *AgentGroup) SearchQuery(opts SearchQueryOptions, cb SearchQueryCallback) (PendingOp, error) {
return ag.clusterAgent.SearchQuery(opts, cb)
}
// DoHTTPRequest will perform an HTTP request against one of the HTTP
// services which are available within the SDK, using a random connected agent.
// If no agent is connected then this will block until one is available or the deadline is reached.
func (ag *AgentGroup) DoHTTPRequest(req *HTTPRequest, cb DoHTTPRequestCallback) (PendingOp, error) {
return ag.clusterAgent.DoHTTPRequest(req, cb)
}
// WaitUntilReady returns whether or not the AgentGroup can ping the requested services.
// This can only be used when no bucket has been opened, if a bucket has been opened then you *must* use the agent
// belonging to that bucket.
func (ag *AgentGroup) WaitUntilReady(deadline time.Time, opts WaitUntilReadyOptions,
cb WaitUntilReadyCallback) (PendingOp, error) {
return ag.clusterAgent.WaitUntilReady(deadline, opts, cb)
}
// Ping pings all of the servers we are connected to and returns
// a report regarding the pings that were performed.
func (ag *AgentGroup) Ping(opts PingOptions, cb PingCallback) (PendingOp, error) {
return ag.clusterAgent.Ping(opts, cb)
}
// Diagnostics returns diagnostics information about the client.
// Mainly containing a list of open connections and their current
// states.
func (ag *AgentGroup) Diagnostics(opts DiagnosticsOptions) (*DiagnosticInfo, error) {
var agents []*Agent
ag.agentsLock.Lock()
// There's no point in trying to get diagnostics from clusterAgent as it has no kv connections.
// In fact it doesn't even expose a Diagnostics function.
for _, agent := range ag.boundAgents {
agents = append(agents, agent)
}
ag.agentsLock.Unlock()
if len(agents) == 0 {
return nil, errors.New("no agents available")
}
var firstError error
var diags []*DiagnosticInfo
for _, agent := range agents {
report, err := agent.diagnostics.Diagnostics(opts)
if err != nil && firstError == nil {
firstError = err
continue
}
diags = append(diags, report)
}
if len(diags) == 0 {
return nil, firstError
}
var overallReport DiagnosticInfo
var connected int
var expected int
for _, report := range diags {
expected++
overallReport.MemdConns = append(overallReport.MemdConns, report.MemdConns...)
if report.State == ClusterStateOnline {
connected++
}
if report.ConfigRev > overallReport.ConfigRev {
overallReport.ConfigRev = report.ConfigRev
}
}
if connected == expected {
overallReport.State = ClusterStateOnline
} else if connected > 0 {
overallReport.State = ClusterStateDegraded
} else {
overallReport.State = ClusterStateOffline
}
return &overallReport, nil
}
func (ag *AgentGroup) maybeCloseGlobalAgent() {
// Close and delete the global level agent that we created on Connect.
agent := ag.boundAgents[""]
if agent == nil {
return
}
logDebugf("Shutting down global level agent")
delete(ag.boundAgents, "")
go func() {
ag.clusterAgent.UnregisterWith(agent.cfgManager, agent.dialer)
if err := agent.Close(); err != nil {
logDebugf("Failed to close agent: %s", err)
}
}()
}
gocbcore-10.2.3/agentgroup_config.go 0000664 0000000 0000000 00000002420 14417540156 0017414 0 ustar 00root root 0000000 0000000 package gocbcore
// AgentGroupConfig specifies the configuration options for creation of an AgentGroup.
type AgentGroupConfig struct {
AgentConfig
}
func (config *AgentGroupConfig) redacted() interface{} {
return config.AgentConfig.redacted()
}
// FromConnStr populates the AgentGroupConfig with information from a
// Couchbase Connection String. See AgentConfig for supported options.
func (config *AgentGroupConfig) FromConnStr(connStr string) error {
return config.AgentConfig.FromConnStr(connStr)
}
func (config *AgentGroupConfig) toAgentConfig() *AgentConfig {
return &AgentConfig{
BucketName: config.BucketName,
UserAgent: config.UserAgent,
SeedConfig: config.SeedConfig,
SecurityConfig: config.SecurityConfig,
CompressionConfig: config.CompressionConfig,
ConfigPollerConfig: config.ConfigPollerConfig,
IoConfig: config.IoConfig,
KVConfig: config.KVConfig,
HTTPConfig: config.HTTPConfig,
DefaultRetryStrategy: config.DefaultRetryStrategy,
CircuitBreakerConfig: config.CircuitBreakerConfig,
OrphanReporterConfig: config.OrphanReporterConfig,
MeterConfig: config.MeterConfig,
TracerConfig: config.TracerConfig,
InternalConfig: config.InternalConfig,
}
}
gocbcore-10.2.3/analyticscomponent.go 0000664 0000000 0000000 00000022243 14417540156 0017633 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"time"
)
// AnalyticsRowReader providers access to the rows of a analytics query
type AnalyticsRowReader struct {
streamer *queryStreamer
statement string
statusCode int
}
// NextRow reads the next rows bytes from the stream
func (q *AnalyticsRowReader) NextRow() []byte {
return q.streamer.NextRow()
}
// Err returns any errors that occurred during streaming.
func (q AnalyticsRowReader) Err() error {
err := q.streamer.Err()
if err != nil {
return err
}
meta, metaErr := q.streamer.MetaData()
if metaErr != nil {
return metaErr
}
raw, descs, err := parseAnalyticsError(meta)
if err != nil {
return &AnalyticsError{
InnerError: err,
Errors: descs,
ErrorText: raw,
Statement: q.statement,
HTTPResponseCode: q.statusCode,
}
}
if len(descs) > 0 {
return &AnalyticsError{
InnerError: errors.New("analytics error"),
Errors: descs,
ErrorText: raw,
Statement: q.statement,
HTTPResponseCode: q.statusCode,
}
}
return nil
}
// MetaData fetches the non-row bytes streamed in the response.
func (q *AnalyticsRowReader) MetaData() ([]byte, error) {
return q.streamer.MetaData()
}
// Close immediately shuts down the connection
func (q *AnalyticsRowReader) Close() error {
return q.streamer.Close()
}
// AnalyticsQueryOptions represents the various options available for an analytics query.
type AnalyticsQueryOptions struct {
Payload []byte
Priority int
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
func wrapAnalyticsError(req *httpRequest, statement string, err error, errBody string, statusCode int) *AnalyticsError {
if err == nil {
err = errors.New("analytics error")
}
ierr := &AnalyticsError{
InnerError: err,
}
if req != nil {
ierr.Endpoint = req.Endpoint
ierr.ClientContextID = req.UniqueID
ierr.RetryAttempts = req.RetryAttempts()
ierr.RetryReasons = req.RetryReasons()
}
ierr.ErrorText = errBody
ierr.Statement = statement
ierr.HTTPResponseCode = statusCode
return ierr
}
type jsonAnalyticsError struct {
Code uint32 `json:"code"`
Msg string `json:"msg"`
}
type jsonAnalyticsErrorResponse struct {
Errors json.RawMessage
}
func parseAnalyticsErrorResp(req *httpRequest, statement string, resp *HTTPResponse) *AnalyticsError {
var errorDescs []AnalyticsErrorDesc
var err error
var raw string
respBody, readErr := ioutil.ReadAll(resp.Body)
if readErr == nil {
raw, errorDescs, err = parseAnalyticsError(respBody)
}
errOut := wrapAnalyticsError(req, statement, err, raw, resp.StatusCode)
errOut.Errors = errorDescs
return errOut
}
func parseAnalyticsError(respBody []byte) (string, []AnalyticsErrorDesc, error) {
var err error
var errorDescs []AnalyticsErrorDesc
var rawRespParse jsonAnalyticsErrorResponse
parseErr := json.Unmarshal(respBody, &rawRespParse)
if parseErr != nil {
return "", nil, nil
}
var respParse []jsonAnalyticsError
parseErr = json.Unmarshal(rawRespParse.Errors, &respParse)
if parseErr == nil {
for _, jsonErr := range respParse {
errorDescs = append(errorDescs, AnalyticsErrorDesc{
Code: jsonErr.Code,
Message: jsonErr.Msg,
})
}
}
if len(errorDescs) >= 1 {
firstErr := errorDescs[0]
errCode := firstErr.Code
errCodeGroup := errCode / 1000
if errCodeGroup == 25 {
err = errInternalServerFailure
}
if errCodeGroup == 20 {
err = errAuthenticationFailure
}
if errCodeGroup == 24 {
err = errCompilationFailure
}
if errCode == 23000 || errCode == 23003 {
err = errTemporaryFailure
}
if errCode == 24000 {
err = errParsingFailure
}
if errCode == 24047 {
err = errIndexNotFound
}
if errCode == 24048 {
err = errIndexExists
}
if errCode == 23007 {
err = errJobQueueFull
}
if errCode == 24025 || errCode == 24044 || errCode == 24045 {
err = errDatasetNotFound
}
if errCode == 24034 {
err = errDataverseNotFound
}
if errCode == 24040 {
err = errDatasetExists
}
if errCode == 24039 {
err = errDataverseExists
}
if errCode == 24006 {
err = errLinkNotFound
}
}
var rawErrors string
if err == nil && len(rawRespParse.Errors) > 0 {
// Only populate if this is an error that we don't recognise.
rawErrors = string(rawRespParse.Errors)
}
return rawErrors, errorDescs, err
}
type analyticsQueryComponent struct {
httpComponent *httpComponent
tracer *tracerComponent
}
func newAnalyticsQueryComponent(httpComponent *httpComponent, tracer *tracerComponent) *analyticsQueryComponent {
return &analyticsQueryComponent{
httpComponent: httpComponent,
tracer: tracer,
}
}
// AnalyticsQuery executes an analytics query
func (aqc *analyticsQueryComponent) AnalyticsQuery(opts AnalyticsQueryOptions, cb AnalyticsQueryCallback) (PendingOp, error) {
tracer := aqc.tracer.StartTelemeteryHandler(metricValueServiceAnalyticsValue, "AnalyticsQuery", opts.TraceContext)
var payloadMap map[string]interface{}
err := json.Unmarshal(opts.Payload, &payloadMap)
if err != nil {
tracer.Finish()
return nil, wrapAnalyticsError(nil, "", wrapError(err, "expected a JSON payload"), "", 0)
}
statement := getMapValueString(payloadMap, "statement", "")
clientContextID := getMapValueString(payloadMap, "client_context_id", "")
readOnly := getMapValueBool(payloadMap, "readonly", false)
ctx, cancel := context.WithCancel(context.Background())
ireq := &httpRequest{
Service: CbasService,
Method: "POST",
Path: "/query/service",
Headers: map[string]string{
"Analytics-Priority": fmt.Sprintf("%d", opts.Priority),
},
Body: opts.Payload,
IsIdempotent: readOnly,
UniqueID: clientContextID,
Deadline: opts.Deadline,
RetryStrategy: opts.RetryStrategy,
RootTraceContext: tracer.RootContext(),
Context: ctx,
CancelFunc: cancel,
User: opts.User,
}
go func() {
res, err := aqc.analyticsQuery(ireq, payloadMap, statement, tracer.StartTime())
if err != nil {
cancel()
tracer.Finish()
cb(nil, err)
return
}
tracer.Finish()
cb(res, nil)
}()
return ireq, nil
}
func (aqc *analyticsQueryComponent) analyticsQuery(ireq *httpRequest, payloadMap map[string]interface{},
statement string, startTime time.Time) (*AnalyticsRowReader, error) {
for {
{
if !ireq.Deadline.IsZero() {
// Produce an updated payload with the appropriate timeout
timeoutLeft := time.Until(ireq.Deadline)
if timeoutLeft <= 0 {
err := &TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "N1QLQuery",
Opaque: ireq.Identifier(),
TimeObserved: time.Since(startTime),
RetryReasons: ireq.retryReasons,
RetryAttempts: ireq.retryCount,
LastDispatchedTo: ireq.Endpoint,
}
return nil, wrapAnalyticsError(ireq, statement, err, "", 0)
}
payloadMap["timeout"] = timeoutLeft.String()
}
newPayload, err := json.Marshal(payloadMap)
if err != nil {
return nil, wrapAnalyticsError(nil, statement, wrapError(err, "failed to produce payload"), "", 0)
}
ireq.Body = newPayload
}
resp, err := aqc.httpComponent.DoInternalHTTPRequest(ireq, false)
if err != nil {
if errors.Is(err, ErrRequestCanceled) {
return nil, err
}
// execHTTPRequest will handle retrying due to in-flight socket close based
// on whether or not IsIdempotent is set on the httpRequest
return nil, wrapAnalyticsError(ireq, statement, err, "", 0)
}
if resp.StatusCode != 200 {
analyticsErr := parseAnalyticsErrorResp(ireq, statement, resp)
var retryReason RetryReason
if len(analyticsErr.Errors) >= 1 {
firstErrDesc := analyticsErr.Errors[0]
if firstErrDesc.Code == 23000 {
retryReason = AnalyticsTemporaryFailureRetryReason
} else if firstErrDesc.Code == 23003 {
retryReason = AnalyticsTemporaryFailureRetryReason
} else if firstErrDesc.Code == 23007 {
retryReason = AnalyticsTemporaryFailureRetryReason
}
}
if retryReason == nil {
// analyticsErr is already wrapped here
return nil, analyticsErr
}
shouldRetry, retryTime := retryOrchMaybeRetry(ireq, retryReason)
if !shouldRetry {
// analyticsErr is already wrapped here
return nil, analyticsErr
}
select {
case <-time.After(time.Until(retryTime)):
continue
case <-time.After(time.Until(ireq.Deadline)):
err := &TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "http",
Opaque: ireq.Identifier(),
TimeObserved: time.Since(startTime),
RetryReasons: ireq.retryReasons,
RetryAttempts: ireq.retryCount,
LastDispatchedTo: ireq.Endpoint,
}
return nil, wrapAnalyticsError(ireq, statement, err, "", 0)
}
}
streamer, err := newQueryStreamer(resp.Body, "results")
if err != nil {
respBody, readErr := ioutil.ReadAll(resp.Body)
if readErr != nil {
logDebugf("Failed to read response body: %v", readErr)
}
return nil, wrapAnalyticsError(ireq, statement, err, string(respBody), resp.StatusCode)
}
return &AnalyticsRowReader{
streamer: streamer,
}, nil
}
}
gocbcore-10.2.3/analyticscomponent_test.go 0000664 0000000 0000000 00000017774 14417540156 0020707 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"testing"
"time"
)
type analyticsTestHelper struct {
TestName string
NumDocs int
TestDocs *testDocs
suite *StandardTestSuite
}
func hlpRunAnalyticsQuery(t *testing.T, agent *AgentGroup, opts AnalyticsQueryOptions) ([][]byte, error) {
t.Helper()
resCh := make(chan *AnalyticsRowReader, 1)
errCh := make(chan error, 1)
_, err := agent.AnalyticsQuery(opts, func(reader *AnalyticsRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
return nil, err
}
var rows *AnalyticsRowReader
select {
case err := <-errCh:
return nil, err
case res := <-resCh:
rows = res
}
var rowBytes [][]byte
for {
row := rows.NextRow()
if row == nil {
break
}
rowBytes = append(rowBytes, row)
}
err = rows.Err()
return rowBytes, err
}
func hlpEnsureDataset(t *testing.T, agent *AgentGroup, bucketName string) {
t.Helper()
payloadStr := fmt.Sprintf("{\"statement\":\"CREATE DATASET IF NOT EXISTS `%s` ON `%s`\"}", bucketName, bucketName)
_, err := hlpRunAnalyticsQuery(t, agent, AnalyticsQueryOptions{
Payload: []byte(payloadStr),
Deadline: time.Now().Add(30000 * time.Millisecond),
})
if err != nil {
t.Logf("Error occurred creating dataset: %s\n", err)
}
payloadStr = "{\"statement\":\"CONNECT LINK Local\"}"
_, err = hlpRunAnalyticsQuery(t, agent, AnalyticsQueryOptions{
Payload: []byte(payloadStr),
Deadline: time.Now().Add(30000 * time.Millisecond),
})
if err != nil {
t.Logf("Error occurred connecting link: %s\n", err)
}
}
func (nqh *analyticsTestHelper) testSetup(t *testing.T) {
agent := nqh.suite.DefaultAgent()
ag := nqh.suite.AgentGroup()
nqh.TestDocs = makeTestDocs(t, agent, nqh.TestName, nqh.NumDocs)
hlpEnsureDataset(t, ag, nqh.suite.BucketName)
}
func (nqh *analyticsTestHelper) testCleanup(t *testing.T) {
if nqh.TestDocs != nil {
nqh.TestDocs.Remove()
nqh.TestDocs = nil
}
}
func (nqh *analyticsTestHelper) testBasic(t *testing.T) {
ag := nqh.suite.AgentGroup()
deadline := time.Now().Add(60000 * time.Millisecond)
runTestQuery := func() ([]testDoc, error) {
test := map[string]interface{}{
"statement": fmt.Sprintf("SELECT i,testName FROM %s WHERE testName=\"%s\"", nqh.suite.BucketName, nqh.TestName),
"client_context_id": "1235",
}
payload, err := json.Marshal(test)
if err != nil {
t.Errorf("failed to marshal test payload: %s", err)
}
iterDeadline := time.Now().Add(5000 * time.Millisecond)
if iterDeadline.After(deadline) {
iterDeadline = deadline
}
resCh := make(chan *AnalyticsRowReader)
errCh := make(chan error)
_, err = ag.AnalyticsQuery(AnalyticsQueryOptions{
Payload: payload,
RetryStrategy: nil,
Deadline: iterDeadline,
}, func(reader *AnalyticsRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
return nil, err
}
var rows *AnalyticsRowReader
select {
case err := <-errCh:
return nil, err
case res := <-resCh:
rows = res
}
var docs []testDoc
for {
row := rows.NextRow()
if row == nil {
break
}
var doc testDoc
err := json.Unmarshal(row, &doc)
if err != nil {
return nil, err
}
docs = append(docs, doc)
}
err = rows.Err()
if err != nil {
return nil, err
}
return docs, nil
}
lastError := ""
for {
docs, err := runTestQuery()
if err == nil {
testFailed := false
for _, doc := range docs {
if doc.I < 1 || doc.I > nqh.NumDocs {
lastError = fmt.Sprintf("query test read invalid row i=%d", doc.I)
testFailed = true
}
}
numDocs := len(docs)
if numDocs != nqh.NumDocs {
lastError = fmt.Sprintf("query test read invalid number of rows %d!=%d", numDocs, 5)
testFailed = true
}
if !testFailed {
break
}
} else {
t.Logf("Error occurred running analytics query, will retry: %s\n", err)
}
sleepDeadline := time.Now().Add(1000 * time.Millisecond)
if sleepDeadline.After(deadline) {
sleepDeadline = deadline
}
time.Sleep(sleepDeadline.Sub(time.Now()))
if sleepDeadline == deadline {
t.Errorf("timed out waiting for indexing: %s", lastError)
break
}
}
}
func (suite *StandardTestSuite) TestAnalytics() {
suite.EnsureSupportsFeature(TestFeatureCbas)
helper := &analyticsTestHelper{
TestName: "testAnalyticsQuery",
NumDocs: 5,
suite: suite,
}
if suite.T().Run("setup", helper.testSetup) {
suite.tracer.Reset()
suite.T().Run("Basic", helper.testBasic)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 1) {
for i := 0; i < len(nilParents); i++ {
suite.AssertHTTPSpan(nilParents[i], "AnalyticsQuery")
}
}
}
suite.VerifyMetrics(suite.meter, "cbas:AnalyticsQuery", 1, true, false)
}
suite.T().Run("cleanup", helper.testCleanup)
}
func (suite *StandardTestSuite) TestAnalyticsCancel() {
suite.EnsureSupportsFeature(TestFeatureCbas)
agent := suite.DefaultAgent()
rt := &roundTripper{delay: 1 * time.Second, tsport: agent.http.cli.Transport}
httpCpt := newHTTPComponentWithClient(
httpComponentProps{},
&http.Client{Transport: rt},
agent.httpMux,
agent.tracer,
)
cbasCpt := newAnalyticsQueryComponent(httpCpt, &tracerComponent{tracer: suite.tracer, metrics: suite.meter})
resCh := make(chan *AnalyticsRowReader)
errCh := make(chan error)
payloadStr := `{"statement":"SELECT * FROM test LIMIT 1","client_context_id":"1235"}`
op, err := cbasCpt.AnalyticsQuery(AnalyticsQueryOptions{
Payload: []byte(payloadStr),
Deadline: time.Now().Add(5 * time.Second),
}, func(reader *AnalyticsRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
suite.T().Fatalf("Failed to execute query %s", err)
}
op.Cancel()
var rows *AnalyticsRowReader
var resErr error
select {
case err := <-errCh:
resErr = err
case res := <-resCh:
rows = res
}
if rows != nil {
suite.T().Fatal("Received rows but should not have")
}
if !errors.Is(resErr, ErrRequestCanceled) {
suite.T().Fatalf("Error should have been request canceled but was %s", resErr)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 1) {
for i := 0; i < len(nilParents); i++ {
suite.AssertHTTPSpan(nilParents[i], "AnalyticsQuery")
}
}
}
suite.VerifyMetrics(suite.meter, "cbas:AnalyticsQuery", 1, false, false)
}
func (suite *StandardTestSuite) TestAnalyticsTimeout() {
suite.EnsureSupportsFeature(TestFeatureCbas)
ag := suite.AgentGroup()
resCh := make(chan *AnalyticsRowReader)
errCh := make(chan error)
payloadStr := fmt.Sprintf(`{"statement":"SELECT * FROM %s LIMIT 1","client_context_id":"12345"}`, suite.BucketName)
_, err := ag.AnalyticsQuery(AnalyticsQueryOptions{
Payload: []byte(payloadStr),
Deadline: time.Now().Add(1 * time.Microsecond),
}, func(reader *AnalyticsRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
suite.T().Fatalf("Failed to execute query %s", err)
}
var rows *AnalyticsRowReader
var resErr error
select {
case err := <-errCh:
resErr = err
case res := <-resCh:
rows = res
}
if rows != nil {
suite.T().Fatal("Received rows but should not have")
}
if !errors.Is(resErr, ErrTimeout) {
suite.T().Fatalf("Error should have been timeout but was %s", resErr)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 1) {
if suite.Assert().Equal(len(nilParents), 1) {
span := nilParents[0]
suite.Assert().Equal("AnalyticsQuery", span.Name)
suite.Assert().Equal(1, len(span.Tags))
suite.Assert().Equal("couchbase", span.Tags["db.system"])
suite.Assert().True(span.Finished)
_, ok := span.Spans[spanNameDispatchToServer]
suite.Assert().False(ok)
}
}
}
suite.VerifyMetrics(suite.meter, "cbas:AnalyticsQuery", 1, false, false)
}
gocbcore-10.2.3/asyncmutex.go 0000664 0000000 0000000 00000003677 14417540156 0016133 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"sync"
)
type asyncMutex struct {
lock sync.Mutex
waiters []func(func())
codeCntr uint64
curCode uint64
}
// acquireLocked grabs the lock and returns its unlock code to be passed to .unlock()
func (q *asyncMutex) acquireLocked() uint64 {
q.codeCntr++
myCode := q.codeCntr
if q.curCode != 0 {
logWarnf("unexpectedly trying to lock asyncMutex while already locked")
}
q.curCode = myCode
return myCode
}
func (q *asyncMutex) Lock(cb func(func())) {
q.lock.Lock()
if q.curCode == 0 {
myCode := q.acquireLocked()
q.lock.Unlock()
cb(func() {
q.unlock(myCode)
})
return
}
q.waiters = append(q.waiters, cb)
q.lock.Unlock()
}
func (q *asyncMutex) LockSync() {
waitCh := make(chan struct{}, 1)
q.Lock(func(unlockFn func()) {
waitCh <- struct{}{}
})
<-waitCh
}
func (q *asyncMutex) UnlockSync() {
// We cheat for sync locks and just grab the code
q.lock.Lock()
syncCode := q.curCode
q.lock.Unlock()
q.unlock(syncCode)
}
func (q *asyncMutex) unlock(myCode uint64) {
q.lock.Lock()
if myCode != q.curCode {
logWarnf("unexpected unlock code for asyncMutex unlock")
q.lock.Unlock()
return
}
q.curCode = 0
if len(q.waiters) == 0 {
q.lock.Unlock()
return
}
nextFn := q.waiters[0]
q.waiters = q.waiters[1:]
nextCode := q.acquireLocked()
q.lock.Unlock()
nextFn(func() {
q.unlock(nextCode)
})
}
gocbcore-10.2.3/asyncwaitqueue.go 0000664 0000000 0000000 00000002363 14417540156 0016771 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"sync"
)
type asyncWaitGroup struct {
lock sync.Mutex
count int
waiters []func()
}
func (q *asyncWaitGroup) IsEmpty() bool {
q.lock.Lock()
isEmpty := q.count == 0
q.lock.Unlock()
return isEmpty
}
func (q *asyncWaitGroup) Add(n int) {
var waiters []func()
q.lock.Lock()
q.count += n
if q.count == 0 {
waiters = q.waiters
q.waiters = nil
}
q.lock.Unlock()
for _, waiter := range waiters {
waiter()
}
}
func (q *asyncWaitGroup) Done() {
q.Add(-1)
}
func (q *asyncWaitGroup) Wait(fn func()) {
q.lock.Lock()
if q.count == 0 {
q.lock.Unlock()
fn()
return
}
q.waiters = append(q.waiters, fn)
q.lock.Unlock()
}
gocbcore-10.2.3/auth.go 0000664 0000000 0000000 00000004244 14417540156 0014663 0 ustar 00root root 0000000 0000000 package gocbcore
import "crypto/tls"
// UserPassPair represents a username and password pair.
type UserPassPair struct {
Username string
Password string
}
// AuthCredsRequest represents an authentication details request from the agent.
type AuthCredsRequest struct {
Service ServiceType
Endpoint string
}
// AuthCertRequest represents a certificate details request from the agent.
type AuthCertRequest struct {
Service ServiceType
Endpoint string
}
// AuthProvider is an interface to allow the agent to fetch authentication
// credentials on-demand from the application.
type AuthProvider interface {
SupportsTLS() bool
SupportsNonTLS() bool
Certificate(req AuthCertRequest) (*tls.Certificate, error)
Credentials(req AuthCredsRequest) ([]UserPassPair, error)
}
func getSingleAuthCreds(auth AuthProvider, req AuthCredsRequest) (UserPassPair, error) {
creds, err := auth.Credentials(req)
if err != nil {
return UserPassPair{}, err
}
if len(creds) != 1 {
return UserPassPair{}, errInvalidCredentials
}
return creds[0], nil
}
func getKvAuthCreds(auth AuthProvider, endpoint string) (UserPassPair, error) {
return getSingleAuthCreds(auth, AuthCredsRequest{
Service: MemdService,
Endpoint: endpoint,
})
}
// PasswordAuthProvider provides a standard AuthProvider implementation
// for use with a standard username/password pair (for example, RBAC).
type PasswordAuthProvider struct {
Username string
Password string
}
// SupportsNonTLS specifies whether this authenticator supports non-TLS connections.
func (auth PasswordAuthProvider) SupportsNonTLS() bool {
return true
}
// SupportsTLS specifies whether this authenticator supports TLS connections.
func (auth PasswordAuthProvider) SupportsTLS() bool {
return true
}
// Certificate directly returns a certificate chain to present for the connection.
func (auth PasswordAuthProvider) Certificate(req AuthCertRequest) (*tls.Certificate, error) {
return nil, nil
}
// Credentials directly returns the username/password from the provider.
func (auth PasswordAuthProvider) Credentials(req AuthCredsRequest) ([]UserPassPair, error) {
return []UserPassPair{{
Username: auth.Username,
Password: auth.Password,
}}, nil
}
gocbcore-10.2.3/authclient.go 0000664 0000000 0000000 00000011302 14417540156 0016053 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"crypto/sha1" // nolint: gosec
"crypto/sha256"
"crypto/sha512"
"hash"
"time"
"github.com/couchbase/gocbcore/v10/memd"
scram "github.com/couchbase/gocbcore/v10/scram"
)
// AuthMechanism represents a type of auth that can be performed.
type AuthMechanism string
const (
// PlainAuthMechanism represents that PLAIN auth should be performed.
PlainAuthMechanism = AuthMechanism("PLAIN")
// ScramSha1AuthMechanism represents that SCRAM SHA1 auth should be performed.
ScramSha1AuthMechanism = AuthMechanism("SCRAM-SHA1")
// ScramSha256AuthMechanism represents that SCRAM SHA256 auth should be performed.
ScramSha256AuthMechanism = AuthMechanism("SCRAM-SHA256")
// ScramSha512AuthMechanism represents that SCRAM SHA512 auth should be performed.
ScramSha512AuthMechanism = AuthMechanism("SCRAM-SHA512")
)
// AuthClient exposes an interface for performing authentication on a
// connected Couchbase K/V client.
type AuthClient interface {
Address() string
SupportsFeature(feature memd.HelloFeature) bool
SaslListMechs(deadline time.Time, cb func(mechs []AuthMechanism, err error)) error
SaslAuth(k, v []byte, deadline time.Time, cb func(b []byte, err error)) error
SaslStep(k, v []byte, deadline time.Time, cb func(err error)) error
}
// SaslListMechsCompleted is used to contain the result and/or error from a SaslListMechs operation.
type SaslListMechsCompleted struct {
Err error
Mechs []AuthMechanism
}
// SaslAuthPlain performs PLAIN SASL authentication against an AuthClient.
func SaslAuthPlain(username, password string, client AuthClient, deadline time.Time, cb func(err error)) error {
// Build PLAIN auth data
userBuf := []byte(username)
passBuf := []byte(password)
authData := make([]byte, 1+len(userBuf)+1+len(passBuf))
authData[0] = 0
copy(authData[1:], userBuf)
authData[1+len(userBuf)] = 0
copy(authData[1+len(userBuf)+1:], passBuf)
// Execute PLAIN authentication
err := client.SaslAuth([]byte(PlainAuthMechanism), authData, deadline, func(b []byte, err error) {
if err != nil {
cb(err)
return
}
cb(nil)
})
if err != nil {
return err
}
return nil
}
func saslAuthScram(saslName []byte, newHash func() hash.Hash, username, password string, client AuthClient,
deadline time.Time, continueCb func(), completedCb func(err error)) error {
scramMgr := scram.NewClient(newHash, username, password)
// Perform the initial SASL step
scramMgr.Step(nil)
err := client.SaslAuth(saslName, scramMgr.Out(), deadline, func(b []byte, err error) {
if err != nil && !isErrorStatus(err, memd.StatusAuthContinue) {
completedCb(err)
return
}
if !scramMgr.Step(b) {
err = scramMgr.Err()
if err != nil {
completedCb(err)
return
}
logErrorf("Local auth client finished before server accepted auth")
completedCb(nil)
return
}
err = client.SaslStep(saslName, scramMgr.Out(), deadline, completedCb)
if err != nil {
completedCb(err)
return
}
continueCb()
})
if err != nil {
return err
}
return nil
}
// SaslAuthScramSha1 performs SCRAM-SHA1 SASL authentication against an AuthClient.
func SaslAuthScramSha1(username, password string, client AuthClient, deadline time.Time, continueCb func(), completedCb func(err error)) error {
return saslAuthScram([]byte("SCRAM-SHA1"), sha1.New, username, password, client, deadline, continueCb, completedCb)
}
// SaslAuthScramSha256 performs SCRAM-SHA256 SASL authentication against an AuthClient.
func SaslAuthScramSha256(username, password string, client AuthClient, deadline time.Time, continueCb func(), completedCb func(err error)) error {
return saslAuthScram([]byte("SCRAM-SHA256"), sha256.New, username, password, client, deadline, continueCb, completedCb)
}
// SaslAuthScramSha512 performs SCRAM-SHA512 SASL authentication against an AuthClient.
func SaslAuthScramSha512(username, password string, client AuthClient, deadline time.Time, continueCb func(), completedCb func(err error)) error {
return saslAuthScram([]byte("SCRAM-SHA512"), sha512.New, username, password, client, deadline, continueCb, completedCb)
}
func saslMethod(method AuthMechanism, username, password string, client AuthClient, deadline time.Time, continueCb func(), completedCb func(err error)) error {
switch method {
case PlainAuthMechanism:
return SaslAuthPlain(username, password, client, deadline, completedCb)
case ScramSha1AuthMechanism:
return SaslAuthScramSha1(username, password, client, deadline, continueCb, completedCb)
case ScramSha256AuthMechanism:
return SaslAuthScramSha256(username, password, client, deadline, continueCb, completedCb)
case ScramSha512AuthMechanism:
return SaslAuthScramSha512(username, password, client, deadline, continueCb, completedCb)
default:
return errNoSupportedMechanisms
}
}
gocbcore-10.2.3/basehttpcfgcontroller.go 0000664 0000000 0000000 00000014670 14417540156 0020324 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"fmt"
"github.com/google/uuid"
"io"
"net/url"
"sync"
"sync/atomic"
"time"
)
type configStreamBlock struct {
Bytes []byte
}
func (i *configStreamBlock) UnmarshalJSON(data []byte) error {
i.Bytes = make([]byte, len(data))
copy(i.Bytes, data)
return nil
}
func hostnameFromURI(uri string) string {
uriInfo, err := url.Parse(uri)
if err != nil {
return uri
}
hostname, err := hostFromHostPort(uriInfo.Host)
if err != nil {
return uri
}
return hostname
}
type baseHTTPConfigController struct {
cfgMgr *configManagementComponent
confHTTPRetryDelay time.Duration
confHTTPRedialPeriod time.Duration
confHTTPMaxWait time.Duration
httpComponent *httpComponent
bucketName string
endpointCallback func(uint64) string
looperStopSig chan struct{}
looperDoneSig chan struct{}
fetchErr error
errLock sync.Mutex
}
type httpPollerProperties struct {
confHTTPRetryDelay time.Duration
confHTTPRedialPeriod time.Duration
confHTTPMaxWait time.Duration
httpComponent *httpComponent
}
func newBaseHTTPConfigController(bucketName string, props httpPollerProperties, cfgMgr *configManagementComponent,
endpointCallback func(uint64) string) *baseHTTPConfigController {
return &baseHTTPConfigController{
cfgMgr: cfgMgr,
confHTTPRedialPeriod: props.confHTTPRedialPeriod,
confHTTPRetryDelay: props.confHTTPRetryDelay,
confHTTPMaxWait: props.confHTTPMaxWait,
httpComponent: props.httpComponent,
bucketName: bucketName,
looperStopSig: make(chan struct{}),
looperDoneSig: make(chan struct{}),
endpointCallback: endpointCallback,
}
}
func (hcc *baseHTTPConfigController) Error() error {
hcc.errLock.Lock()
defer hcc.errLock.Unlock()
return hcc.fetchErr
}
func (hcc *baseHTTPConfigController) setError(err error) {
hcc.errLock.Lock()
hcc.fetchErr = err
hcc.errLock.Unlock()
}
func (hcc *baseHTTPConfigController) Done() chan struct{} {
return hcc.looperDoneSig
}
func (hcc *baseHTTPConfigController) Stop() {
close(hcc.looperStopSig)
}
// Reset must never be called concurrently with the Stop or whilst the poll loop is running.
func (hcc *baseHTTPConfigController) Reset() {
hcc.looperStopSig = make(chan struct{})
hcc.looperDoneSig = make(chan struct{})
}
func (hcc *baseHTTPConfigController) DoLoop() {
hcc.doLoop()
close(hcc.looperDoneSig)
}
func (hcc *baseHTTPConfigController) doLoop() {
waitPeriod := hcc.confHTTPRetryDelay
maxConnPeriod := hcc.confHTTPRedialPeriod
var iterNum uint64 = 1
iterSawConfig := false
logDebugf("HTTP Looper starting.")
for {
select {
case <-hcc.looperStopSig:
return
default:
}
pickedSrv := hcc.endpointCallback(iterNum)
if pickedSrv == "" {
logDebugf("Pick Failed.")
// All servers have been visited during this iteration
if !iterSawConfig {
logDebugf("Looper waiting...")
// Wait for a period before trying again if there was a problem...
// We also watch for the client being shut down.
select {
case <-hcc.looperStopSig:
return
case <-time.After(waitPeriod):
}
}
logDebugf("Looping again.")
// Go to next iteration and try all servers again
iterNum++
iterSawConfig = false
continue
}
logDebugf("Http Picked: %s.", pickedSrv)
hostname := hostnameFromURI(pickedSrv)
logDebugf("HTTP Hostname: %s.", hostname)
var resp *HTTPResponse
// 1 on success, 0 on failure for node, -1 for generic failure
var doConfigRequest func(bool) int
doConfigRequest = func(is2x bool) int {
streamPath := "bs"
if is2x {
streamPath = "bucketsStreaming"
}
// HTTP request time!
uri := fmt.Sprintf("/pools/default/%s/%s", streamPath, url.PathEscape(hcc.bucketName))
logDebugf("Requesting config from: %s/%s.", pickedSrv, uri)
req := &httpRequest{
Service: MgmtService,
Method: "GET",
Path: uri,
Endpoint: pickedSrv,
UniqueID: uuid.New().String(),
Deadline: time.Now().Add(hcc.confHTTPMaxWait),
}
var err error
resp, err = hcc.httpComponent.DoInternalHTTPRequest(req, true)
if err != nil {
logWarnf("Failed to connect to host. %v", err)
hcc.setError(err)
return 0
}
if resp.StatusCode != 200 {
err := resp.Body.Close()
if err != nil {
logErrorf("Socket close failed handling status code != 200 (%s)", err)
}
if resp.StatusCode == 401 {
logWarnf("Failed to connect to host, bad auth.")
hcc.setError(errAuthenticationFailure)
return -1
} else if resp.StatusCode == 404 {
if is2x {
logWarnf("Failed to connect to host, bad bucket.")
hcc.setError(errAuthenticationFailure)
return -1
}
return doConfigRequest(true)
}
logWarnf("Failed to connect to host, unexpected status code: %v.", resp.StatusCode)
hcc.setError(errCliInternalError)
return 0
}
hcc.setError(nil)
return 1
}
switch doConfigRequest(false) {
case 0:
continue
case -1:
continue
}
logDebugf("Connected.")
var autoDisconnected int32
// Autodisconnect eventually
go func() {
select {
case <-time.After(maxConnPeriod):
case <-hcc.looperStopSig:
}
logDebugf("Automatically resetting our HTTP connection")
atomic.StoreInt32(&autoDisconnected, 1)
err := resp.Body.Close()
if err != nil {
logErrorf("Socket close failed during auto-dc (%s)", err)
}
}()
dec := json.NewDecoder(resp.Body)
configBlock := new(configStreamBlock)
for {
err := dec.Decode(configBlock)
if err != nil {
if atomic.LoadInt32(&autoDisconnected) == 1 {
// If we know we intentionally disconnected, we know we do not
// need to close the client, nor log an error, since this was
// expected behaviour
break
}
logWarnf("Config block decode failure (%s)", err)
if err != io.EOF {
err = resp.Body.Close()
if err != nil {
logErrorf("Socket close failed after decode fail (%s)", err)
}
}
break
}
logDebugf("Got Block: %v", string(configBlock.Bytes))
bkCfg, err := parseConfig(configBlock.Bytes, hostname)
if err != nil {
logDebugf("Got error while parsing config: %v", err)
err = resp.Body.Close()
if err != nil {
logErrorf("Socket close failed after parsing fail (%s)", err)
}
break
}
logDebugf("Got Config.")
iterSawConfig = true
logDebugf("HTTP Config Update")
hcc.cfgMgr.OnNewConfig(bkCfg)
}
logDebugf("HTTP, Setting %s to iter %d", pickedSrv, iterNum)
}
}
gocbcore-10.2.3/capella_ca.go 0000664 0000000 0000000 00000002234 14417540156 0015763 0 ustar 00root root 0000000 0000000 package gocbcore
var capellaRootCA = []byte(`
-----BEGIN CERTIFICATE-----
MIIDFTCCAf2gAwIBAgIRANLVkgOvtaXiQJi0V6qeNtswDQYJKoZIhvcNAQELBQAw
JDESMBAGA1UECgwJQ291Y2hiYXNlMQ4wDAYDVQQLDAVDbG91ZDAeFw0xOTEyMDYy
MjEyNTlaFw0yOTEyMDYyMzEyNTlaMCQxEjAQBgNVBAoMCUNvdWNoYmFzZTEOMAwG
A1UECwwFQ2xvdWQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCfvOIi
enG4Dp+hJu9asdxEMRmH70hDyMXv5ZjBhbo39a42QwR59y/rC/sahLLQuNwqif85
Fod1DkqgO6Ng3vecSAwyYVkj5NKdycQu5tzsZkghlpSDAyI0xlIPSQjoORA/pCOU
WOpymA9dOjC1bo6rDyw0yWP2nFAI/KA4Z806XeqLREuB7292UnSsgFs4/5lqeil6
rL3ooAw/i0uxr/TQSaxi1l8t4iMt4/gU+W52+8Yol0JbXBTFX6itg62ppb/Eugmn
mQRMgL67ccZs7cJ9/A0wlXencX2ohZQOR3mtknfol3FH4+glQFn27Q4xBCzVkY9j
KQ20T1LgmGSngBInAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE
FJQOBPvrkU2In1Sjoxt97Xy8+cKNMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0B
AQsFAAOCAQEARgM6XwcXPLSpFdSf0w8PtpNGehmdWijPM3wHb7WZiS47iNen3oq8
m2mm6V3Z57wbboPpfI+VEzbhiDcFfVnK1CXMC0tkF3fnOG1BDDvwt4jU95vBiNjY
xdzlTP/Z+qr0cnVbGBSZ+fbXstSiRaaAVcqQyv3BRvBadKBkCyPwo+7svQnScQ5P
Js7HEHKVms5tZTgKIw1fbmgR2XHleah1AcANB+MAPBCcTgqurqr5G7W2aPSBLLGA
fRIiVzm7VFLc7kWbp7ENH39HVG6TZzKnfl9zJYeiklo5vQQhGSMhzBsO70z4RRzi
DPFAN/4qZAgD5q3AFNIq2WWADFQGSwVJhg==
-----END CERTIFICATE-----`)
gocbcore-10.2.3/cbcrc.go 0000664 0000000 0000000 00000006605 14417540156 0015001 0 ustar 00root root 0000000 0000000 package gocbcore
var crc32tab = []uint32{
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,
0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,
0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,
0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,
0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,
0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,
0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,
0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,
0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,
0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,
0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,
0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,
0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,
0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,
0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d}
func cbCrc(key []byte) uint32 {
crc := uint32(0xffffffff)
for x := 0; x < len(key); x++ {
crc = (crc >> 8) ^ crc32tab[(uint64(crc)^uint64(key[x]))&0xff]
}
return (^crc) >> 16 & 0x7fff
}
func cbcVbMap(key []byte, numVbs uint32) uint16 {
return uint16(cbCrc(key) % numVbs)
}
gocbcore-10.2.3/cbcrc_test.go 0000664 0000000 0000000 00000000566 14417540156 0016040 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"fmt"
"strings"
"testing"
)
func TestGenerateIDs(t *testing.T) {
var ids []string
prefix := "rangecollectionretry"
var counter int
for {
id := fmt.Sprintf("%s-%d", prefix, counter)
counter++
if cbCrc([]byte(id)) == 12 {
ids = append(ids, id)
}
if len(ids) == 10 {
fmt.Println(strings.Join(ids, "\",\""))
return
}
}
}
gocbcore-10.2.3/cccpcfgcontroller.go 0000664 0000000 0000000 00000013711 14417540156 0017415 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"math/rand"
"sync"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
type cccpConfigController struct {
muxer dispatcher
cfgMgr *configManagementComponent
confCccpPollPeriod time.Duration
confCccpMaxWait time.Duration
looperStopSig chan struct{}
looperDoneSig chan struct{}
fetchErr error
errLock sync.Mutex
isFallbackErrorFn func(error) bool
noConfigFoundFn func(error)
}
func newCCCPConfigController(props cccpPollerProperties, muxer dispatcher, cfgMgr *configManagementComponent,
isFallbackErrorFn func(error) bool, noConfigFoundFn func(error)) *cccpConfigController {
return &cccpConfigController{
muxer: muxer,
cfgMgr: cfgMgr,
confCccpPollPeriod: props.confCccpPollPeriod,
confCccpMaxWait: props.confCccpMaxWait,
looperStopSig: make(chan struct{}),
looperDoneSig: make(chan struct{}),
isFallbackErrorFn: isFallbackErrorFn,
noConfigFoundFn: noConfigFoundFn,
}
}
type cccpPollerProperties struct {
confCccpPollPeriod time.Duration
confCccpMaxWait time.Duration
}
func (ccc *cccpConfigController) Error() error {
ccc.errLock.Lock()
defer ccc.errLock.Unlock()
return ccc.fetchErr
}
func (ccc *cccpConfigController) setError(err error) {
ccc.errLock.Lock()
ccc.fetchErr = err
ccc.errLock.Unlock()
}
func (ccc *cccpConfigController) Stop() {
close(ccc.looperStopSig)
}
func (ccc *cccpConfigController) Done() chan struct{} {
return ccc.looperDoneSig
}
// Reset must never be called concurrently with the Stop or whilst the poll loop is running.
func (ccc *cccpConfigController) Reset() {
ccc.looperStopSig = make(chan struct{})
ccc.looperDoneSig = make(chan struct{})
}
func (ccc *cccpConfigController) DoLoop() error {
if err := ccc.doLoop(); err != nil {
return err
}
close(ccc.looperDoneSig)
return nil
}
func (ccc *cccpConfigController) doLoop() error {
tickTime := ccc.confCccpPollPeriod
logInfof("CCCP Looper starting.")
nodeIdx := -1
// The first time that we loop we want to skip any sleep so that we can try get a config and bootstrapped ASAP.
firstLoop := true
for {
if !firstLoop {
// Wait for either the agent to be shut down, or our tick time to expire
select {
case <-ccc.looperStopSig:
return nil
case <-time.After(tickTime):
}
}
firstLoop = false
iter, err := ccc.muxer.PipelineSnapshot()
if err != nil {
// If we have an error it indicates the client is shut down.
break
}
numNodes := iter.NumPipelines()
if numNodes == 0 {
logInfof("CCCPPOLL: No nodes available to poll, returning upstream")
return errNoCCCPHosts
}
if nodeIdx < 0 || nodeIdx > numNodes {
nodeIdx = rand.Intn(numNodes) // #nosec G404
}
var foundConfig *cfgBucket
var fallbackErr error
var wasCancelled bool
iter.Iterate(nodeIdx, func(pipeline *memdPipeline) bool {
nodeIdx = (nodeIdx + 1) % numNodes
cccpBytes, err := ccc.getClusterConfig(pipeline)
if err != nil {
if ccc.isFallbackErrorFn(err) {
fallbackErr = err
return false
}
// Only log the error at warn if it's unexpected.
// If we cancelled the request or we're shutting down the connection then it's not really unexpected.
if errors.Is(err, ErrRequestCanceled) || errors.Is(err, ErrShutdown) {
wasCancelled = true
logDebugf("CCCPPOLL: CCCP request was cancelled or connection was shutdown: %v", err)
return true
}
// This error is checked by WaitUntilReady when no config has been seen.
ccc.setError(err)
logWarnf("CCCPPOLL: Failed to retrieve CCCP config. %s", err)
return false
}
fallbackErr = nil
ccc.setError(nil)
logDebugf("CCCPPOLL: Got Block: %s", string(cccpBytes))
hostName, err := hostFromHostPort(pipeline.Address())
if err != nil {
logWarnf("CCCPPOLL: Failed to parse source address. %s", err)
return false
}
bk, err := parseConfig(cccpBytes, hostName)
if err != nil {
logWarnf("CCCPPOLL: Failed to parse CCCP config. %v", err)
return false
}
foundConfig = bk
return true
})
if fallbackErr != nil {
// This error is indicative of a memcached bucket which we can't handle so return the error.
logInfof("CCCPPOLL: CCCP not supported, returning error upstream.")
return fallbackErr
}
if foundConfig == nil {
// Only log the error at warn if it's unexpected.
// If we cancelled the request then we're shutting down or request was requeued and this isn't unexpected.
if wasCancelled {
logDebugf("CCCPPOLL: CCCP request was cancelled.")
} else {
logWarnf("CCCPPOLL: Failed to retrieve config from any node.")
ccc.noConfigFoundFn(err)
}
continue
}
logDebugf("CCCPPOLL: Received new config")
ccc.cfgMgr.OnNewConfig(foundConfig)
}
return nil
}
func (ccc *cccpConfigController) getClusterConfig(pipeline *memdPipeline) (cfgOut []byte, errOut error) {
signal := make(chan struct{}, 1)
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGetClusterConfig,
},
Callback: func(resp *memdQResponse, _ *memdQRequest, err error) {
if resp != nil {
cfgOut = resp.Packet.Value
}
errOut = err
signal <- struct{}{}
},
RetryStrategy: newFailFastRetryStrategy(),
}
err := pipeline.SendRequest(req)
if err != nil {
return nil, err
}
timeoutTmr := AcquireTimer(ccc.confCccpMaxWait)
select {
case <-signal:
ReleaseTimer(timeoutTmr, false)
return
case <-timeoutTmr.C:
ReleaseTimer(timeoutTmr, true)
// We've timed out so lets check underlying connections to see if they're responsible.
clients := pipeline.Clients()
for _, cli := range clients {
err := cli.Error()
if err != nil {
logDebugf("Found error in pipeline client %p/%s: %v", cli, cli.address, err)
req.cancelWithCallback(err)
<-signal
return
}
}
req.cancelWithCallback(errUnambiguousTimeout)
<-signal
return
case <-ccc.looperStopSig:
ReleaseTimer(timeoutTmr, false)
req.cancelWithCallback(errRequestCanceled)
<-signal
return
}
}
gocbcore-10.2.3/circuitbreaker.go 0000664 0000000 0000000 00000014064 14417540156 0016721 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"sync/atomic"
"time"
)
const (
circuitBreakerStateDisabled uint32 = iota
circuitBreakerStateClosed
circuitBreakerStateHalfOpen
circuitBreakerStateOpen
)
type circuitBreaker interface {
AllowsRequest() bool
MarkSuccessful()
MarkFailure()
State() uint32
Reset()
CanaryTimeout() time.Duration
CompletionCallback(error) bool
}
// CircuitBreakerCallback is the callback used by the circuit breaker to determine if an error should count toward
// the circuit breaker failure count.
type CircuitBreakerCallback func(error) bool
// CircuitBreakerConfig is the set of configuration settings for configuring circuit breakers.
// If Disabled is set to true then a noop circuit breaker will be used, otherwise a lazy circuit
// breaker.
type CircuitBreakerConfig struct {
Enabled bool
// VolumeThreshold is the minimum amount of operations to measure before the threshold percentage kicks in.
VolumeThreshold int64
// ErrorThresholdPercentage is the percentage of operations that need to fail in a window until the circuit opens.
ErrorThresholdPercentage float64
// SleepWindow is the initial sleep time after which a canary is sent as a probe.
SleepWindow time.Duration
// RollingWindow is the rolling timeframe which is used to calculate the error threshold percentage.
RollingWindow time.Duration
// CompletionCallback is called on every response to determine if it is successful or not.
CompletionCallback CircuitBreakerCallback
// CanaryTimeout is the timeout for the canary request until it is deemed failed.
CanaryTimeout time.Duration
}
type noopCircuitBreaker struct {
}
func newNoopCircuitBreaker() *noopCircuitBreaker {
return &noopCircuitBreaker{}
}
func (ncb *noopCircuitBreaker) AllowsRequest() bool {
return true
}
func (ncb *noopCircuitBreaker) MarkSuccessful() {
}
func (ncb *noopCircuitBreaker) MarkFailure() {
}
func (ncb *noopCircuitBreaker) State() uint32 {
return circuitBreakerStateDisabled
}
func (ncb *noopCircuitBreaker) Reset() {
}
func (ncb *noopCircuitBreaker) CompletionCallback(error) bool {
return true
}
func (ncb *noopCircuitBreaker) CanaryTimeout() time.Duration {
return 0
}
type lazyCircuitBreaker struct {
windowStart int64
sleepWindow int64
rollingWindow int64
volumeThreshold int64
errorPercentageThreshold float64
canaryTimeout time.Duration
total int64
failed int64
openedAt int64
sendCanaryFn func()
completionCallback CircuitBreakerCallback
state uint32
}
func newLazyCircuitBreaker(config CircuitBreakerConfig, canaryFn func()) *lazyCircuitBreaker {
if config.VolumeThreshold == 0 {
config.VolumeThreshold = 20
}
if config.ErrorThresholdPercentage == 0 {
config.ErrorThresholdPercentage = 50
}
if config.SleepWindow == 0 {
config.SleepWindow = 5 * time.Second
}
if config.RollingWindow == 0 {
config.RollingWindow = 1 * time.Minute
}
if config.CanaryTimeout == 0 {
config.CanaryTimeout = 5 * time.Second
}
if config.CompletionCallback == nil {
config.CompletionCallback = func(err error) bool {
return !errors.Is(err, ErrTimeout)
}
}
breaker := &lazyCircuitBreaker{
sleepWindow: int64(config.SleepWindow * time.Nanosecond),
rollingWindow: int64(config.RollingWindow * time.Nanosecond),
volumeThreshold: config.VolumeThreshold,
errorPercentageThreshold: config.ErrorThresholdPercentage,
canaryTimeout: config.CanaryTimeout,
sendCanaryFn: canaryFn,
completionCallback: config.CompletionCallback,
}
breaker.Reset()
return breaker
}
func (lcb *lazyCircuitBreaker) Reset() {
now := time.Now().UnixNano()
atomic.StoreUint32(&lcb.state, circuitBreakerStateClosed)
atomic.StoreInt64(&lcb.total, 0)
atomic.StoreInt64(&lcb.failed, 0)
atomic.StoreInt64(&lcb.openedAt, 0)
atomic.StoreInt64(&lcb.windowStart, now)
}
func (lcb *lazyCircuitBreaker) State() uint32 {
return atomic.LoadUint32(&lcb.state)
}
func (lcb *lazyCircuitBreaker) AllowsRequest() bool {
state := lcb.State()
if state == circuitBreakerStateClosed {
return true
}
elapsed := (time.Now().UnixNano() - atomic.LoadInt64(&lcb.openedAt)) > lcb.sleepWindow
if elapsed && atomic.CompareAndSwapUint32(&lcb.state, circuitBreakerStateOpen, circuitBreakerStateHalfOpen) {
// If we're outside of the sleep window and the circuit is open then send a canary.
go lcb.sendCanaryFn()
}
return false
}
func (lcb *lazyCircuitBreaker) MarkSuccessful() {
if atomic.CompareAndSwapUint32(&lcb.state, circuitBreakerStateHalfOpen, circuitBreakerStateClosed) {
logDebugf("Moving circuit breaker to closed")
lcb.Reset()
return
}
lcb.maybeResetRollingWindow()
atomic.AddInt64(&lcb.total, 1)
}
func (lcb *lazyCircuitBreaker) MarkFailure() {
now := time.Now().UnixNano()
if atomic.CompareAndSwapUint32(&lcb.state, circuitBreakerStateHalfOpen, circuitBreakerStateOpen) {
logDebugf("Moving circuit breaker from half open to open")
atomic.StoreInt64(&lcb.openedAt, now)
return
}
lcb.maybeResetRollingWindow()
atomic.AddInt64(&lcb.total, 1)
atomic.AddInt64(&lcb.failed, 1)
lcb.maybeOpenCircuit()
}
func (lcb *lazyCircuitBreaker) CanaryTimeout() time.Duration {
return lcb.canaryTimeout
}
func (lcb *lazyCircuitBreaker) CompletionCallback(err error) bool {
return lcb.completionCallback(err)
}
func (lcb *lazyCircuitBreaker) maybeOpenCircuit() {
if atomic.LoadInt64(&lcb.total) < lcb.volumeThreshold {
return
}
currentPercentage := (float64(atomic.LoadInt64(&lcb.failed)) / float64(atomic.LoadInt64(&lcb.total))) * 100
if currentPercentage >= lcb.errorPercentageThreshold {
logDebugf("Moving circuit breaker to open")
atomic.StoreUint32(&lcb.state, circuitBreakerStateOpen)
atomic.StoreInt64(&lcb.openedAt, time.Now().UnixNano())
}
}
func (lcb *lazyCircuitBreaker) maybeResetRollingWindow() {
now := time.Now().UnixNano()
if (now - atomic.LoadInt64(&lcb.windowStart)) <= lcb.rollingWindow {
return
}
atomic.StoreInt64(&lcb.windowStart, now)
atomic.StoreInt64(&lcb.total, 0)
atomic.StoreInt64(&lcb.failed, 0)
}
gocbcore-10.2.3/circuitbreaker_test.go 0000664 0000000 0000000 00000012632 14417540156 0017757 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"sync/atomic"
"time"
)
func (suite *StandardTestSuite) TestLazyCircuitBreakerSuccessfulCanary() {
var canarySent int32
var breaker *lazyCircuitBreaker
breaker = newLazyCircuitBreaker(CircuitBreakerConfig{
VolumeThreshold: 4,
ErrorThresholdPercentage: 60,
SleepWindow: 10 * time.Millisecond,
RollingWindow: 70 * time.Millisecond,
}, func() {
atomic.StoreInt32(&canarySent, 1)
breaker.MarkSuccessful()
})
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkSuccessful()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkSuccessful()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkFailure()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkFailure()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkFailure()
if breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should not have allowed request")
}
// Give time for the sleep window to expire
time.Sleep(20 * time.Millisecond)
if breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should not have allowed request")
}
// Give time for the canary to be sent
time.Sleep(10 * time.Millisecond)
if atomic.LoadInt32(&canarySent) != 1 {
suite.T().Fatalf("Circuit breaker should have sent canary")
}
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
// Give time for rolling window to reset.
time.Sleep(100 * time.Millisecond)
breaker.MarkSuccessful()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
}
func (suite *StandardTestSuite) TestLazyCircuitBreakerFailedCanary() {
var canarySent int32
var breaker *lazyCircuitBreaker
breaker = newLazyCircuitBreaker(CircuitBreakerConfig{
VolumeThreshold: 4,
ErrorThresholdPercentage: 60,
SleepWindow: 10 * time.Millisecond,
RollingWindow: 70 * time.Millisecond,
}, func() {
atomic.StoreInt32(&canarySent, 1)
breaker.MarkFailure()
})
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkSuccessful()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkSuccessful()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkFailure()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkFailure()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkFailure()
if breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should not have allowed request")
}
// Give time for the sleep window to expire.
time.Sleep(20 * time.Millisecond)
if breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should not have allowed request")
}
// Give time for the canary to be sent.
time.Sleep(10 * time.Millisecond)
if atomic.LoadInt32(&canarySent) != 1 {
suite.T().Fatalf("Circuit breaker should not have sent canary")
}
if breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should not have allowed request")
}
// Give time for rolling window to reset.
time.Sleep(100 * time.Millisecond)
breaker.MarkSuccessful()
if breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should not have allowed request")
}
}
func (suite *StandardTestSuite) TestLazyCircuitBreakerReset() {
var canarySent int32
var breaker *lazyCircuitBreaker
breaker = newLazyCircuitBreaker(CircuitBreakerConfig{
VolumeThreshold: 4,
ErrorThresholdPercentage: 60,
SleepWindow: 10 * time.Millisecond,
RollingWindow: 1 * time.Second,
}, func() {
atomic.StoreInt32(&canarySent, 1)
breaker.MarkFailure()
})
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkFailure()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkFailure()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkFailure()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
breaker.MarkFailure()
if breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should not have allowed request")
}
breaker.Reset()
// Give time for the sleep window to expire, in this case we expect things to have been reset
// so nothing should have happened.
time.Sleep(20 * time.Millisecond)
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
// Give time for the canary to be sent
time.Sleep(10 * time.Millisecond)
if atomic.LoadInt32(&canarySent) != 0 {
suite.T().Fatalf("Circuit breaker should not have sent canary")
}
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
// Give time for rolling window to reset.
time.Sleep(100 * time.Millisecond)
breaker.MarkSuccessful()
if !breaker.AllowsRequest() {
suite.T().Fatalf("Circuit breaker should have allowed request")
}
}
gocbcore-10.2.3/clusteragent.go 0000664 0000000 0000000 00000017075 14417540156 0016430 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"fmt"
"sync"
"time"
)
type clusterAgent struct {
defaultRetryStrategy RetryStrategy
httpMux *httpMux
tracer *tracerComponent
http *httpComponent
diagnostics *diagnosticsComponent
n1ql *n1qlQueryComponent
analytics *analyticsQueryComponent
search *searchQueryComponent
views *viewQueryComponent
revLock sync.Mutex
revID int64
configWatchLock sync.Mutex
configWatchers []routeConfigWatcher
}
func createClusterAgent(config *clusterAgentConfig) (*clusterAgent, error) {
tracer := config.TracerConfig.Tracer
if tracer == nil {
tracer = noopTracer{}
}
tracerCmpt := newTracerComponent(tracer, "", config.TracerConfig.NoRootTraceSpans, config.MeterConfig.Meter)
c := &clusterAgent{
tracer: tracerCmpt,
defaultRetryStrategy: config.DefaultRetryStrategy,
}
tlsConfig, err := setupTLSConfig(config.SeedConfig.MemdAddrs, config.SecurityConfig)
if err != nil {
return nil, err
}
if c.defaultRetryStrategy == nil {
c.defaultRetryStrategy = newFailFastRetryStrategy()
}
circuitBreakerConfig := config.CircuitBreakerConfig
userAgent := config.UserAgent
httpEpList := routeEndpoints{}
for _, hostPort := range config.SeedConfig.HTTPAddrs {
if config.SecurityConfig.UseTLS && !config.SecurityConfig.NoTLSSeedNode {
ep := routeEndpoint{
Address: fmt.Sprintf("https://%s", hostPort),
IsSeedNode: true,
}
httpEpList.SSLEndpoints = append(httpEpList.SSLEndpoints, ep)
} else {
ep := routeEndpoint{
Address: fmt.Sprintf("http://%s", hostPort),
IsSeedNode: true,
}
httpEpList.NonSSLEndpoints = append(httpEpList.NonSSLEndpoints, ep)
}
}
c.httpMux = newHTTPMux(
circuitBreakerConfig,
c,
&httpClientMux{tlsConfig: tlsConfig, auth: config.SecurityConfig.Auth},
config.SecurityConfig.NoTLSSeedNode,
)
c.http = newHTTPComponent(
httpComponentProps{
UserAgent: userAgent,
DefaultRetryStrategy: c.defaultRetryStrategy,
},
httpClientProps{
maxIdleConns: config.HTTPConfig.MaxIdleConns,
maxIdleConnsPerHost: config.HTTPConfig.MaxIdleConnsPerHost,
idleTimeout: config.HTTPConfig.IdleConnectionTimeout,
},
c.httpMux,
c.tracer,
)
c.n1ql = newN1QLQueryComponent(c.http, c, c.tracer)
c.analytics = newAnalyticsQueryComponent(c.http, c.tracer)
c.search = newSearchQueryComponent(c.http, c.tracer)
c.views = newViewQueryComponent(c.http, c.tracer)
// diagnostics at this level will never need to hook KV. There are no persistent connections
// so Diagnostics calls should be blocked. Ping and WaitUntilReady will only try HTTP services.
c.diagnostics = newDiagnosticsComponent(nil, c.httpMux, c.http, "", c.defaultRetryStrategy, nil)
// Kick everything off.
cfg := &routeConfig{
mgmtEpList: httpEpList,
revID: -1,
}
c.httpMux.OnNewRouteConfig(cfg)
return c, nil
}
func (agent *clusterAgent) RegisterWith(cfgMgr configManager, dialer *memdClientDialerComponent) {
cfgMgr.AddConfigWatcher(agent)
dialer.AddBootstrapFailHandler(agent.diagnostics)
}
func (agent *clusterAgent) UnregisterWith(cfgMgr configManager, dialer *memdClientDialerComponent) {
cfgMgr.RemoveConfigWatcher(agent)
dialer.RemoveBootstrapFailHandler(agent.diagnostics)
}
func (agent *clusterAgent) AddConfigWatcher(watcher routeConfigWatcher) {
agent.configWatchLock.Lock()
agent.configWatchers = append(agent.configWatchers, watcher)
agent.configWatchLock.Unlock()
}
func (agent *clusterAgent) RemoveConfigWatcher(watcher routeConfigWatcher) {
var idx int
agent.configWatchLock.Lock()
for i, w := range agent.configWatchers {
if w == watcher {
idx = i
}
}
if idx == len(agent.configWatchers) {
agent.configWatchers = agent.configWatchers[:idx]
} else {
agent.configWatchers = append(agent.configWatchers[:idx], agent.configWatchers[idx+1:]...)
}
agent.configWatchLock.Unlock()
}
func (agent *clusterAgent) OnNewRouteConfig(cfg *routeConfig) {
agent.revLock.Lock()
// This could be coming from multiple agents so we need to make sure that it's up to date with what we've seen.
if cfg.revID > -1 && cfg.revID <= agent.revID {
agent.revLock.Unlock()
return
}
logDebugf("Cluster agent applying config rev id: %d\n", cfg.revID)
agent.revID = cfg.revID
agent.revLock.Unlock()
agent.configWatchLock.Lock()
watchers := agent.configWatchers
agent.configWatchLock.Unlock()
for _, watcher := range watchers {
watcher.OnNewRouteConfig(cfg)
}
}
// N1QLQuery executes a N1QL query against a random connected agent.
func (agent *clusterAgent) N1QLQuery(opts N1QLQueryOptions, cb N1QLQueryCallback) (PendingOp, error) {
return agent.n1ql.N1QLQuery(opts, cb)
}
// PreparedN1QLQuery executes a prepared N1QL query against a random connected agent.
func (agent *clusterAgent) PreparedN1QLQuery(opts N1QLQueryOptions, cb N1QLQueryCallback) (PendingOp, error) {
return agent.n1ql.PreparedN1QLQuery(opts, cb)
}
// AnalyticsQuery executes an analytics query against a random connected agent.
func (agent *clusterAgent) AnalyticsQuery(opts AnalyticsQueryOptions, cb AnalyticsQueryCallback) (PendingOp, error) {
return agent.analytics.AnalyticsQuery(opts, cb)
}
// SearchQuery executes a Search query against a random connected agent.
func (agent *clusterAgent) SearchQuery(opts SearchQueryOptions, cb SearchQueryCallback) (PendingOp, error) {
return agent.search.SearchQuery(opts, cb)
}
// ViewQuery executes a view query against a random connected agent.
func (agent *clusterAgent) ViewQuery(opts ViewQueryOptions, cb ViewQueryCallback) (PendingOp, error) {
return agent.views.ViewQuery(opts, cb)
}
// DoHTTPRequest will perform an HTTP request against one of the HTTP
// services which are available within the SDK, using a random connected agent.
func (agent *clusterAgent) DoHTTPRequest(req *HTTPRequest, cb DoHTTPRequestCallback) (PendingOp, error) {
return agent.http.DoHTTPRequest(req, cb)
}
// Ping pings all of the servers we are connected to and returns
// a report regarding the pings that were performed.
func (agent *clusterAgent) Ping(opts PingOptions, cb PingCallback) (PendingOp, error) {
for _, srv := range opts.ServiceTypes {
if srv == MemdService {
return nil, wrapError(errInvalidArgument, "memd service is not valid for use with clusterAgent")
} else if srv == CapiService {
return nil, wrapError(errInvalidArgument, "capi service is not valid for use with clusterAgent")
}
}
if len(opts.ServiceTypes) == 0 {
opts.ServiceTypes = []ServiceType{CbasService, FtsService, N1qlService, MgmtService}
opts.ignoreMissingServices = true
}
return agent.diagnostics.Ping(opts, cb)
}
// WaitUntilReady returns whether or not the Agent has seen a valid cluster config.
func (agent *clusterAgent) WaitUntilReady(deadline time.Time, opts WaitUntilReadyOptions, cb WaitUntilReadyCallback) (PendingOp, error) {
for _, srv := range opts.ServiceTypes {
if srv == MemdService {
return nil, wrapError(errInvalidArgument, "memd service is not valid for use with clusterAgent")
} else if srv == CapiService {
return nil, wrapError(errInvalidArgument, "capi service is not valid for use with clusterAgent")
}
}
forceWait := true
if len(opts.ServiceTypes) == 0 {
forceWait = false
opts.ServiceTypes = []ServiceType{CbasService, FtsService, N1qlService, MgmtService}
}
return agent.diagnostics.WaitUntilReady(deadline, forceWait, opts, cb)
}
// Close shuts down the agent, closing the underlying http client. This does not cause the agent
// to unregister itself with any configuration providers so be sure to do that first.
func (agent *clusterAgent) Close() error {
// Close the transports so that they don't hold open goroutines.
agent.http.Close()
return nil
}
gocbcore-10.2.3/clusteragent_config.go 0000664 0000000 0000000 00000001120 14417540156 0017735 0 ustar 00root root 0000000 0000000 package gocbcore
type clusterAgentConfig struct {
UserAgent string
SeedConfig SeedConfig
SecurityConfig SecurityConfig
HTTPConfig HTTPConfig
TracerConfig TracerConfig
MeterConfig MeterConfig
DefaultRetryStrategy RetryStrategy
CircuitBreakerConfig CircuitBreakerConfig
}
func (config *clusterAgentConfig) redacted() interface{} {
newConfig := *config
if isLogRedactionLevelFull() {
// The slices here are still pointing at config's underlying arrays
// so we need to make them not do that.
newConfig.SeedConfig = newConfig.SeedConfig.redacted()
}
return newConfig
}
gocbcore-10.2.3/collections.go 0000664 0000000 0000000 00000007320 14417540156 0016236 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"strconv"
"time"
)
const (
unknownCid = uint32(0xFFFFFFFF)
pendingCid = uint32(0xFFFFFFFE)
)
// ManifestCollection is the representation of a collection within a manifest.
type ManifestCollection struct {
UID uint32
Name string
MaxTTL uint32
}
// UnmarshalJSON is a custom implementation of json unmarshaling.
func (item *ManifestCollection) UnmarshalJSON(data []byte) error {
decData := struct {
UID string `json:"uid"`
Name string `json:"name"`
MaxTTL uint32 `json:"maxTTL"`
}{}
if err := json.Unmarshal(data, &decData); err != nil {
return err
}
decUID, err := strconv.ParseUint(decData.UID, 16, 32)
if err != nil {
return err
}
item.UID = uint32(decUID)
item.Name = decData.Name
item.MaxTTL = decData.MaxTTL
return nil
}
// ManifestScope is the representation of a scope within a manifest.
type ManifestScope struct {
UID uint32
Name string
Collections []ManifestCollection
}
// UnmarshalJSON is a custom implementation of json unmarshaling.
func (item *ManifestScope) UnmarshalJSON(data []byte) error {
decData := struct {
UID string `json:"uid"`
Name string `json:"name"`
Collections []ManifestCollection `json:"collections"`
}{}
if err := json.Unmarshal(data, &decData); err != nil {
return err
}
decUID, err := strconv.ParseUint(decData.UID, 16, 32)
if err != nil {
return err
}
item.UID = uint32(decUID)
item.Name = decData.Name
item.Collections = decData.Collections
return nil
}
// Manifest is the representation of a collections manifest.
type Manifest struct {
UID uint64
Scopes []ManifestScope
}
// UnmarshalJSON is a custom implementation of json unmarshaling.
func (item *Manifest) UnmarshalJSON(data []byte) error {
decData := struct {
UID string `json:"uid"`
Scopes []ManifestScope `json:"scopes"`
}{}
if err := json.Unmarshal(data, &decData); err != nil {
return err
}
decUID, err := strconv.ParseUint(decData.UID, 16, 64)
if err != nil {
return err
}
item.UID = decUID
item.Scopes = decData.Scopes
return nil
}
// GetCollectionManifestOptions are the options available to the GetCollectionManifest command.
type GetCollectionManifestOptions struct {
TraceContext RequestSpanContext
RetryStrategy RetryStrategy
Deadline time.Time
}
// GetAllCollectionManifestsOptions are the options available to the GetAllCollectionManifests command.
type GetAllCollectionManifestsOptions struct {
TraceContext RequestSpanContext
RetryStrategy RetryStrategy
Deadline time.Time
}
// GetCollectionIDOptions are the options available to the GetCollectionID command.
type GetCollectionIDOptions struct {
RetryStrategy RetryStrategy
TraceContext RequestSpanContext
Deadline time.Time
}
// GetCollectionIDResult encapsulates the result of a GetCollectionID operation.
type GetCollectionIDResult struct {
ManifestID uint64
CollectionID uint32
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// GetCollectionManifestResult encapsulates the result of a GetCollectionManifest operation.
type GetCollectionManifestResult struct {
Manifest []byte
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// SingleServerManifestResult encapsulates the result from a single server when using the GetAllCollectionManifests
// operation.
type SingleServerManifestResult struct {
Manifest []byte
Error error
}
// GetAllCollectionManifestsResult encapsulates the result of a GetAllCollectionManifests operation.
type GetAllCollectionManifestsResult struct {
Manifests map[string]SingleServerManifestResult
}
gocbcore-10.2.3/collectionscomponent.go 0000664 0000000 0000000 00000043224 14417540156 0020164 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func (cidMgr *collectionsComponent) createKey(scopeName, collectionName string) string {
return fmt.Sprintf("%s.%s", scopeName, collectionName)
}
type collectionsComponent struct {
idMap map[string]*collectionIDCache
mapLock sync.Mutex
dispatcher dispatcher
maxQueueSize int
tracer *tracerComponent
defaultRetryStrategy RetryStrategy
cfgMgr configManager
// pendingOpQueue is used when collections are enabled but we've not yet seen a cluster config to confirm
// whether or not collections are supported.
pendingOpQueue *memdOpQueue
configSeen uint32
}
type collectionIDProps struct {
MaxQueueSize int
DefaultRetryStrategy RetryStrategy
}
func newCollectionIDManager(props collectionIDProps, dispatcher dispatcher, tracer *tracerComponent,
cfgMgr configManager) *collectionsComponent {
cidMgr := &collectionsComponent{
dispatcher: dispatcher,
idMap: make(map[string]*collectionIDCache),
maxQueueSize: props.MaxQueueSize,
tracer: tracer,
defaultRetryStrategy: props.DefaultRetryStrategy,
cfgMgr: cfgMgr,
pendingOpQueue: newMemdOpQueue(),
}
cfgMgr.AddConfigWatcher(cidMgr)
dispatcher.SetPostCompleteErrorHandler(cidMgr.handleOpRoutingResp)
return cidMgr
}
func (cidMgr *collectionsComponent) OnNewRouteConfig(cfg *routeConfig) {
if !atomic.CompareAndSwapUint32(&cidMgr.configSeen, 0, 1) {
return
}
colsSupported := cfg.ContainsBucketCapability("collections")
cidMgr.cfgMgr.RemoveConfigWatcher(cidMgr)
cidMgr.pendingOpQueue.Close()
cidMgr.pendingOpQueue.Drain(func(request *memdQRequest) {
// Anything in this queue is here because collections were present so if we definitely don't support collections
// then fail them.
if !colsSupported {
request.tryCallback(nil, errCollectionsUnsupported)
return
}
cidMgr.requeue(request)
})
}
func (cidMgr *collectionsComponent) handleCollectionUnknown(req *memdQRequest) bool {
// We cannot retry requests with no collection information.
// This also prevents the GetCollectionID requests from being automatically retried.
if req.CollectionName == "" && req.ScopeName == "" {
return false
}
shouldRetry, retryTime := retryOrchMaybeRetry(req, KVCollectionOutdatedRetryReason)
if shouldRetry {
go func() {
time.Sleep(time.Until(retryTime))
cidMgr.requeue(req)
}()
}
return shouldRetry
}
func (cidMgr *collectionsComponent) handleOpRoutingResp(resp *memdQResponse, req *memdQRequest, err error) (bool, error) {
if errors.Is(err, ErrCollectionNotFound) {
if cidMgr.handleCollectionUnknown(req) {
return true, nil
}
}
return false, err
}
func (cidMgr *collectionsComponent) GetCollectionManifest(opts GetCollectionManifestOptions, cb GetCollectionManifestCallback) (PendingOp, error) {
tracer := cidMgr.tracer.StartTelemeteryHandler(metricValueServiceAnalyticsValue, "GetCollectionManifest", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
cb(nil, err)
tracer.Finish()
return
}
res := GetCollectionManifestResult{
Manifest: resp.Value,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(&res, nil)
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = cidMgr.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdCollectionsGetManifest,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: nil,
Value: nil,
},
Callback: handler,
RetryStrategy: opts.RetryStrategy,
RootTraceContext: opts.TraceContext,
}
op, err := cidMgr.dispatcher.DispatchDirect(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallbackAndFinishTracer(&TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "GetCollectionManifest",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
}, tracer)
}))
}
return op, nil
}
func (cidMgr *collectionsComponent) GetAllCollectionManifests(opts GetAllCollectionManifestsOptions, cb GetAllCollectionManifestsCallback) (PendingOp, error) {
tracer := cidMgr.tracer.StartTelemeteryHandler(metricValueServiceAnalyticsValue, "GetAllCollectionManifests", opts.TraceContext)
if opts.RetryStrategy == nil {
opts.RetryStrategy = cidMgr.defaultRetryStrategy
}
iter, err := cidMgr.dispatcher.PipelineSnapshot()
if err != nil {
tracer.Finish()
return nil, err
}
manifests := make(map[string]SingleServerManifestResult)
manifestsLock := sync.Mutex{}
op := &multiPendingOp{
isIdempotent: true,
}
opCompleteLocked := func() {
completed := op.IncrementCompletedOps()
if iter.NumPipelines()-int(completed) == 0 {
tracer.Finish()
cb(&GetAllCollectionManifestsResult{Manifests: manifests}, nil)
}
}
var setTimer func(request *memdQRequest)
if opts.Deadline.IsZero() {
setTimer = func(_ *memdQRequest) {}
} else {
start := time.Now()
timeout := opts.Deadline.Sub(start)
setTimer = func(req *memdQRequest) {
req.SetTimer(time.AfterFunc(timeout, func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallbackAndFinishTracer(&TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "GetAllCollectionManifests",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
}, tracer)
}))
}
}
iter.Iterate(0, func(pipeline *memdPipeline) bool {
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
manifestsLock.Lock()
res := SingleServerManifestResult{
Error: err,
}
if resp != nil {
res.Manifest = resp.Value
}
manifests[pipeline.address] = res
opCompleteLocked()
manifestsLock.Unlock()
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdCollectionsGetManifest,
},
Callback: handler,
RetryStrategy: opts.RetryStrategy,
RootTraceContext: opts.TraceContext,
}
curOp, err := cidMgr.dispatcher.DispatchDirectToAddress(req, pipeline)
if err == nil {
setTimer(req)
op.ops = append(op.ops, curOp)
return false
}
manifestsLock.Lock()
manifests[pipeline.address] = SingleServerManifestResult{Error: err}
opCompleteLocked()
manifestsLock.Unlock()
return false
})
return op, nil
}
// GetCollectionID does not trigger retries on unknown collection. This is because the request sets the scope and collection
// name in the key rather than in the corresponding fields.
func (cidMgr *collectionsComponent) GetCollectionID(scopeName string, collectionName string, opts GetCollectionIDOptions,
cb GetCollectionIDCallback) (PendingOp, error) {
tracer := cidMgr.tracer.StartTelemeteryHandler(metricValueServiceAnalyticsValue, "GetCollectionID", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
manifestID := binary.BigEndian.Uint64(resp.Extras[0:])
collectionID := binary.BigEndian.Uint32(resp.Extras[8:])
cidMgr.upsert(scopeName, collectionName, collectionID)
res := GetCollectionIDResult{
ManifestID: manifestID,
CollectionID: collectionID,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(&res, nil)
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = cidMgr.defaultRetryStrategy
}
keyScopeName := scopeName
if keyScopeName == "" {
keyScopeName = "_default"
}
keyCollectionName := collectionName
if keyCollectionName == "" {
keyCollectionName = "_default"
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdCollectionsGetID,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: nil,
Value: []byte(fmt.Sprintf("%s.%s", keyScopeName, keyCollectionName)),
Vbucket: 0,
},
ReplicaIdx: -1,
RetryStrategy: opts.RetryStrategy,
RootTraceContext: opts.TraceContext,
}
req.Callback = handler
op, err := cidMgr.dispatcher.DispatchDirect(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallbackAndFinishTracer(&TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "GetCollectionID",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
}, tracer)
}))
}
return op, nil
}
func (cidMgr *collectionsComponent) upsert(scopeName, collectionName string, value uint32) *collectionIDCache {
cidMgr.mapLock.Lock()
id, ok := cidMgr.idMap[cidMgr.createKey(scopeName, collectionName)]
if !ok {
id = cidMgr.newCollectionIDCache(scopeName, collectionName)
key := cidMgr.createKey(scopeName, collectionName)
cidMgr.idMap[key] = id
}
id.lock.Lock()
id.setID(value)
id.lock.Unlock()
cidMgr.mapLock.Unlock()
return id
}
func (cidMgr *collectionsComponent) getAndMaybeInsert(scopeName, collectionName string, value uint32) *collectionIDCache {
cidMgr.mapLock.Lock()
id, ok := cidMgr.idMap[cidMgr.createKey(scopeName, collectionName)]
if !ok {
id = cidMgr.newCollectionIDCache(scopeName, collectionName)
id.lock.Lock()
id.setID(value)
id.lock.Unlock()
key := cidMgr.createKey(scopeName, collectionName)
cidMgr.idMap[key] = id
}
cidMgr.mapLock.Unlock()
return id
}
func (cidMgr *collectionsComponent) remove(scopeName, collectionName string) {
logDebugf("Removing cache entry for %s.%s", scopeName, collectionName)
cidMgr.mapLock.Lock()
delete(cidMgr.idMap, cidMgr.createKey(scopeName, collectionName))
cidMgr.mapLock.Unlock()
}
func (cidMgr *collectionsComponent) newCollectionIDCache(scope, collection string) *collectionIDCache {
return &collectionIDCache{
dispatcher: cidMgr.dispatcher,
maxQueueSize: cidMgr.maxQueueSize,
parent: cidMgr,
scopeName: scope,
collectionName: collection,
}
}
type collectionIDCache struct {
opQueue *memdOpQueue
id uint32
collectionName string
scopeName string
parent *collectionsComponent
dispatcher dispatcher
lock sync.Mutex
maxQueueSize int
}
func (cid *collectionIDCache) sendWithCid(req *memdQRequest) error {
cid.lock.Lock()
id := cid.id
cid.lock.Unlock()
if err := setRequestCid(req, id); err != nil {
logDebugf("Failed to set collection ID on request: %v", err)
return err
}
_, err := cid.dispatcher.DispatchDirect(req)
if err != nil {
return err
}
return nil
}
func (cid *collectionIDCache) queueRequest(req *memdQRequest) error {
cid.lock.Lock()
defer cid.lock.Unlock()
return cid.opQueue.Push(req, cid.maxQueueSize)
}
func (cid *collectionIDCache) setID(id uint32) {
logDebugf("Setting cache ID to %d for %s.%s", id, cid.scopeName, cid.collectionName)
cid.id = id
}
func (cid *collectionIDCache) refreshCid(req *memdQRequest) error {
err := cid.opQueue.Push(req, cid.maxQueueSize)
if err != nil {
return err
}
logDebugf("Refreshing collection ID for %s.%s", req.ScopeName, req.CollectionName)
_, err = cid.parent.GetCollectionID(req.ScopeName, req.CollectionName, GetCollectionIDOptions{TraceContext: req.RootTraceContext},
func(result *GetCollectionIDResult, err error) {
if err != nil {
if errors.Is(err, ErrCollectionNotFound) {
// The collection is unknown so we need to mark the cid unknown and attempt to retry the request.
// Retrying the request will requeue it in the cid manager so either it will pick up the unknown cid
// and cause a refresh or another request will and this one will get queued within the cache.
// Either the collection will eventually come online or this request will timeout.
logDebugf("Collection %s.%s not found, attempting retry", req.ScopeName, req.CollectionName)
cid.lock.Lock()
cid.setID(unknownCid)
cid.lock.Unlock()
if cid.opQueue.Remove(req) {
if cid.parent.handleCollectionUnknown(req) {
return
}
} else {
logDebugf("Request no longer existed in op queue, possibly cancelled?",
req.Opaque, req.CollectionName)
}
} else {
logDebugf("Collection ID refresh failed: %v", err)
}
// There was an error getting this collection ID so lets remove the cache from the manager and try to
// callback on all of the queued requests.
cid.parent.remove(req.ScopeName, req.CollectionName)
cid.opQueue.Close()
cid.opQueue.Drain(func(request *memdQRequest) {
request.tryCallback(nil, err)
})
return
}
// We successfully got the cid, the GetCollectionID itself will have handled setting the ID on this cache,
// so lets reset the op queue and requeue all of our requests.
logDebugf("Collection %s.%s refresh succeeded, requeuing requests", req.ScopeName, req.CollectionName)
cid.lock.Lock()
opQueue := cid.opQueue
cid.opQueue = newMemdOpQueue()
cid.lock.Unlock()
opQueue.Close()
opQueue.Drain(func(request *memdQRequest) {
request.AddResourceUnitsFromUnitResult(result.Internal.ResourceUnits)
if err := setRequestCid(request, result.CollectionID); err != nil {
logDebugf("Failed to set collection ID on request: %v", err)
request.cancelWithCallback(err)
return
}
cid.dispatcher.RequeueDirect(request, false)
})
},
)
return err
}
func (cid *collectionIDCache) dispatch(req *memdQRequest) error {
cid.lock.Lock()
// if the cid is unknown then mark the request pending and refresh cid first
// if it's pending then queue the request
// otherwise send the request
switch cid.id {
case unknownCid:
logDebugf("Collection %s.%s unknown, refreshing id", req.ScopeName, req.CollectionName)
cid.setID(pendingCid)
cid.opQueue = newMemdOpQueue()
// We attempt to send the refresh inside of the lock, that way we haven't released the lock and allowed an op
// to get queued if we need to move the status back to unknown. Without doing this it's possible for one or
// more op(s) to sneak into the queue and then no more requests come in and those sit in the queue until they
// timeout because nothing is triggering the cid refresh.
err := cid.refreshCid(req)
if err != nil {
// We've failed to send the cid refresh so we need to set it back to unknown otherwise it'll never
// get updated.
cid.setID(unknownCid)
cid.lock.Unlock()
return err
}
cid.lock.Unlock()
return nil
case pendingCid:
logDebugf("Collection %s.%s pending, queueing request OP=0x%x", req.ScopeName, req.CollectionName, req.Command)
cid.lock.Unlock()
return cid.queueRequest(req)
default:
cid.lock.Unlock()
return cid.sendWithCid(req)
}
}
func (cidMgr *collectionsComponent) Dispatch(req *memdQRequest) (PendingOp, error) {
noCollection := req.CollectionName == "" && req.ScopeName == ""
defaultCollection := req.CollectionName == "_default" && req.ScopeName == "_default"
collectionIDPresent := req.CollectionID > 0
// If the user didn't enable collections then we can just not bother with any collections logic.
if !cidMgr.dispatcher.CollectionsEnabled() {
if !(noCollection || defaultCollection) || collectionIDPresent {
return nil, errCollectionsUnsupported
}
_, err := cidMgr.dispatcher.DispatchDirect(req)
if err != nil {
return nil, err
}
return req, nil
}
if noCollection || defaultCollection || collectionIDPresent {
return cidMgr.dispatcher.DispatchDirect(req)
}
if atomic.LoadUint32(&cidMgr.configSeen) == 0 {
logDebugf("Collections are enabled but we've not yet seen a config so queueing request")
err := cidMgr.pendingOpQueue.Push(req, cidMgr.maxQueueSize)
if err != nil {
return nil, err
}
return req, nil
}
if !cidMgr.dispatcher.SupportsCollections() {
return nil, errCollectionsUnsupported
}
cidCache := cidMgr.getAndMaybeInsert(req.ScopeName, req.CollectionName, unknownCid)
err := cidCache.dispatch(req)
if err != nil {
return nil, err
}
return req, nil
}
func (cidMgr *collectionsComponent) requeue(req *memdQRequest) {
cidCache := cidMgr.getAndMaybeInsert(req.ScopeName, req.CollectionName, unknownCid)
cidCache.lock.Lock()
if cidCache.id != unknownCid && cidCache.id != pendingCid {
cidCache.setID(unknownCid)
}
cidCache.lock.Unlock()
err := cidCache.dispatch(req)
if err != nil {
req.tryCallback(nil, err)
}
}
func setRequestCid(req *memdQRequest, cid uint32) error {
if req.Command == memd.CmdRangeScanCreate {
var createReq *rangeScanCreateRequest
if err := json.Unmarshal(req.Value, &createReq); err != nil {
return err
}
createReq.Collection = strconv.FormatUint(uint64(cid), 16)
value, err := json.Marshal(createReq)
if err != nil {
return err
}
req.Value = value
return nil
}
req.CollectionID = cid
return nil
}
gocbcore-10.2.3/collectionscomponent_test.go 0000664 0000000 0000000 00000067075 14417540156 0021235 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/binary"
"errors"
"fmt"
"time"
"github.com/couchbase/gocbcore/v10/memd"
"github.com/stretchr/testify/mock"
)
// When the SDK starts up collections support is unknown.
// This test is for the scenario when a request is made whilst collections support is unknown
// but collections are enabled and the server does support them.
// We should see the SDK queue the request until collections support is known, a request should
// be made to get the collection ID for the collection name and then the user's request sent with
// the collection ID on it.
func (suite *UnitTestSuite) TestCollectionsComponentCollectionsStateUnknownSupported() {
cName := "test"
sName := "_default"
cfgMgr := new(mockConfigManager)
cfgMgr.On("AddConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
cfgMgr.On("RemoveConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
dispatcher := new(mockDispatcher)
dispatcher.On("SetPostCompleteErrorHandler", mock.AnythingOfType("gocbcore.postCompleteErrorHandler")).Return()
dispatcher.On("CollectionsEnabled").Return(true).Once()
dispatcher.On("DispatchDirect", mock.AnythingOfType("*gocbcore.memdQRequest")).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdCollectionsGetID, req.Command)
suite.Assert().Equal([]byte(fmt.Sprintf("%s.%s", sName, cName)), req.Value)
suite.Assert().Empty(req.Key)
suite.Assert().Empty(req.CollectionName)
suite.Assert().Empty(req.ScopeName)
suite.Assert().Equal(-1, req.ReplicaIdx)
extras := make([]byte, 12)
binary.BigEndian.PutUint64(extras[0:], 1)
binary.BigEndian.PutUint32(extras[8:], 8)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{Extras: extras}}, req, nil)
})
})
dispatcher.On("RequeueDirect", mock.AnythingOfType("*gocbcore.memdQRequest"), false).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdGet, req.Command)
suite.Assert().Equal([]byte("test-key"), req.Key)
suite.Assert().Equal(cName, req.CollectionName)
suite.Assert().Equal(sName, req.ScopeName)
suite.Assert().Equal(uint32(8), req.CollectionID)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{Value: []byte("test")}}, req, nil)
})
})
cidMgr := newCollectionIDManager(collectionIDProps{
DefaultRetryStrategy: &failFastRetryStrategy{},
MaxQueueSize: 100},
dispatcher,
newTracerComponent(&noopTracer{}, "", true, &noopMeter{}),
cfgMgr,
)
waitCh := make(chan error, 1)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
waitCh <- err
}
// This request should get queued as the manager hasn't seen a config.
op, err := cidMgr.Dispatch(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: []byte("test-key"),
Value: nil,
},
CollectionName: cName,
ScopeName: sName,
Callback: handler,
RootTraceContext: noopSpanContext{},
})
suite.Require().Nil(err, err)
suite.Assert().NotNil(op)
// Update the cidMgr with a config to dequeue the request.
cidMgr.OnNewRouteConfig(&routeConfig{
bucketCapabilities: []string{"collections"},
})
// Requeueing on cid unknown is done in a go routine
select {
case <-time.After(1 * time.Second):
suite.T().Fatalf("Timed out waiting for callback to be called")
case err := <-waitCh:
suite.Assert().Nil(err, err)
}
cfgMgr.AssertExpectations(suite.T())
dispatcher.AssertExpectations(suite.T())
}
// When the SDK starts up collections support is unknown.
// This test is for the scenario when a request is made whilst collections support is unknown
// but collections are enabled and the server does support them but the collection is initially unknown.
// We should see the SDK queue the request until collections support is known. A request should
// be made to get the collection ID for the collection name twice and then the user's request sent with
// the collection ID on it.
func (suite *UnitTestSuite) TestCollectionsComponentCollectionsStateUnknownCollectionUnknown() {
cName := "test"
sName := "_default"
cfgMgr := new(mockConfigManager)
cfgMgr.On("AddConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
cfgMgr.On("RemoveConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
dispatcher := new(mockDispatcher)
dispatcher.On("SetPostCompleteErrorHandler", mock.AnythingOfType("gocbcore.postCompleteErrorHandler")).Return()
dispatcher.On("CollectionsEnabled").Return(true).Once()
// First request we reply collection unknown.
dispatcher.On("DispatchDirect", mock.AnythingOfType("*gocbcore.memdQRequest")).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdCollectionsGetID, req.Command)
suite.Assert().Equal([]byte(fmt.Sprintf("%s.%s", sName, cName)), req.Value)
suite.Assert().Empty(req.Key)
suite.Assert().Empty(req.CollectionName)
suite.Assert().Empty(req.ScopeName)
suite.Assert().Equal(-1, req.ReplicaIdx)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{}}, req, errCollectionNotFound)
})
}).Once()
// Second request we simulate the collection coming online.
dispatcher.On("DispatchDirect", mock.AnythingOfType("*gocbcore.memdQRequest")).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdCollectionsGetID, req.Command)
suite.Assert().Equal([]byte(fmt.Sprintf("%s.%s", sName, cName)), req.Value)
suite.Assert().Empty(req.Key)
suite.Assert().Empty(req.CollectionName)
suite.Assert().Empty(req.ScopeName)
suite.Assert().Equal(-1, req.ReplicaIdx)
extras := make([]byte, 12)
binary.BigEndian.PutUint64(extras[0:], 1)
binary.BigEndian.PutUint32(extras[8:], 8)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{Extras: extras}}, req, nil)
})
}).Once()
dispatcher.On("RequeueDirect", mock.AnythingOfType("*gocbcore.memdQRequest"), false).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdGet, req.Command)
suite.Assert().Equal([]byte("test-key"), req.Key)
suite.Assert().Equal(cName, req.CollectionName)
suite.Assert().Equal(sName, req.ScopeName)
suite.Assert().Equal(uint32(8), req.CollectionID)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{Value: []byte("test")}}, req, nil)
})
}).Once()
cidMgr := newCollectionIDManager(collectionIDProps{
DefaultRetryStrategy: &failFastRetryStrategy{},
MaxQueueSize: 100},
dispatcher,
newTracerComponent(&noopTracer{}, "", true, &noopMeter{}),
cfgMgr,
)
waitCh := make(chan error, 1)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
waitCh <- err
}
// This request should get queued as the manager hasn't seen a config.
op, err := cidMgr.Dispatch(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: []byte("test-key"),
Value: nil,
},
CollectionName: cName,
ScopeName: sName,
Callback: handler,
RootTraceContext: noopSpanContext{},
})
suite.Require().Nil(err, err)
suite.Assert().NotNil(op)
// Update the cidMgr with a config to dequeue the request.
cidMgr.OnNewRouteConfig(&routeConfig{
bucketCapabilities: []string{"collections"},
})
// Requeueing on cid unknown is done in a go routine
select {
case <-time.After(1 * time.Second):
suite.T().Fatalf("Timed out waiting for callback to be called")
case err := <-waitCh:
suite.Assert().Nil(err, err)
}
cfgMgr.AssertExpectations(suite.T())
dispatcher.AssertExpectations(suite.T())
}
// When the SDK starts up collections support is unknown.
// This test is for the scenario when a request is made whilst collections support is unknown
// but collections are enabled and the server does support them but the cid request is met with a server error.
// We should see the SDK queue the request until the get cid request fails and then the callback should be hit.
func (suite *UnitTestSuite) TestCollectionsComponentCollectionsStateUnknownGenericError() {
cName := "test"
sName := "_default"
cfgMgr := new(mockConfigManager)
cfgMgr.On("AddConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
cfgMgr.On("RemoveConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
dispatcher := new(mockDispatcher)
dispatcher.On("SetPostCompleteErrorHandler", mock.AnythingOfType("gocbcore.postCompleteErrorHandler")).Return()
dispatcher.On("CollectionsEnabled").Return(true).Once()
// First request we reply collection unknown.
dispatcher.On("DispatchDirect", mock.AnythingOfType("*gocbcore.memdQRequest")).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdCollectionsGetID, req.Command)
suite.Assert().Equal([]byte(fmt.Sprintf("%s.%s", sName, cName)), req.Value)
suite.Assert().Empty(req.Key)
suite.Assert().Empty(req.CollectionName)
suite.Assert().Empty(req.ScopeName)
suite.Assert().Equal(-1, req.ReplicaIdx)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{}}, req, errInternalServerFailure)
})
}).Once()
cidMgr := newCollectionIDManager(collectionIDProps{
DefaultRetryStrategy: &failFastRetryStrategy{},
MaxQueueSize: 100},
dispatcher,
newTracerComponent(&noopTracer{}, "", true, &noopMeter{}),
cfgMgr,
)
waitCh := make(chan error, 1)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
waitCh <- err
}
// This request should get queued as the manager hasn't seen a config.
op, err := cidMgr.Dispatch(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: []byte("test-key"),
Value: nil,
},
CollectionName: cName,
ScopeName: sName,
Callback: handler,
RootTraceContext: noopSpanContext{},
})
suite.Require().Nil(err, err)
suite.Assert().NotNil(op)
// Update the cidMgr with a config to dequeue the request.
cidMgr.OnNewRouteConfig(&routeConfig{
bucketCapabilities: []string{"collections"},
})
select {
case <-time.After(1 * time.Second):
suite.T().Fatalf("Timed out waiting for callback to be called")
case err := <-waitCh:
suite.Assert().NotNil(err, err)
}
cfgMgr.AssertExpectations(suite.T())
dispatcher.AssertExpectations(suite.T())
}
// When the SDK starts up collections support is unknown.
// This test is for the scenario when a request is made whilst collections support is unknown
// but collections are enabled and the server does not support them.
// We should see the SDK queue the request until collections support is known.
// The SDK should then fire the request callback with an error.
func (suite *UnitTestSuite) TestCollectionsComponentCollectionsStateUnknownUnsupported() {
cName := "test"
sName := "_default"
cfgMgr := new(mockConfigManager)
cfgMgr.On("AddConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
cfgMgr.On("RemoveConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
dispatcher := new(mockDispatcher)
dispatcher.On("SetPostCompleteErrorHandler", mock.AnythingOfType("gocbcore.postCompleteErrorHandler")).Return()
dispatcher.On("CollectionsEnabled").Return(true).Once()
cidMgr := newCollectionIDManager(collectionIDProps{
DefaultRetryStrategy: &failFastRetryStrategy{},
MaxQueueSize: 100},
dispatcher,
newTracerComponent(&noopTracer{}, "", true, &noopMeter{}),
cfgMgr,
)
var called bool
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
called = true
if !errors.Is(err, ErrCollectionsUnsupported) {
suite.T().Errorf("Error should have been collections unsupported but was: %v", err)
}
}
// This request should get queued as the manager hasn't seen a config.
op, err := cidMgr.Dispatch(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: []byte("test-key"),
Value: nil,
},
CollectionName: cName,
ScopeName: sName,
Callback: handler,
RootTraceContext: noopSpanContext{},
})
suite.Require().Nil(err, err)
suite.Assert().NotNil(op)
// Update the cidMgr with a config to dequeue the request.
cidMgr.OnNewRouteConfig(&routeConfig{
bucketCapabilities: []string{},
})
cfgMgr.AssertExpectations(suite.T())
dispatcher.AssertExpectations(suite.T())
suite.Assert().True(called)
}
// This tests that when the SDK knows the server collections state is unsupported then
// we receive an error.
func (suite *UnitTestSuite) TestCollectionsComponentCollectionsUnsupported() {
cName := "test"
sName := "_default"
cfgMgr := new(mockConfigManager)
cfgMgr.On("AddConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
dispatcher := new(mockDispatcher)
dispatcher.On("SetPostCompleteErrorHandler", mock.AnythingOfType("gocbcore.postCompleteErrorHandler")).Return()
dispatcher.On("CollectionsEnabled").Return(true).Once()
dispatcher.On("SupportsCollections").Return(false).Once()
cidMgr := newCollectionIDManager(collectionIDProps{
DefaultRetryStrategy: &failFastRetryStrategy{},
MaxQueueSize: 100},
dispatcher,
newTracerComponent(&noopTracer{}, "", true, &noopMeter{}),
cfgMgr,
)
cidMgr.configSeen = 1
var called bool
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
called = true
}
// This request should get queued as the manager hasn't seen a config.
op, err := cidMgr.Dispatch(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: []byte("test-key"),
Value: nil,
},
CollectionName: cName,
ScopeName: sName,
Callback: handler,
RootTraceContext: noopSpanContext{},
})
if !errors.Is(err, ErrCollectionsUnsupported) {
suite.T().Errorf("Error should have been collections unsupported but was: %v", err)
}
suite.Assert().Nil(op)
// Update the cidMgr with a config to dequeue the request.
cidMgr.OnNewRouteConfig(&routeConfig{
bucketCapabilities: []string{},
})
cfgMgr.AssertExpectations(suite.T())
dispatcher.AssertExpectations(suite.T())
suite.Assert().False(called)
}
// This test is for the scenario when a request is made whilst collections support is known
// amd collections are enabled, the server does support them, and the collection exists.
// We should see the SDK send a request to get the collection ID for the collection name and
// then the user's request sent with the collection ID on it.
func (suite *UnitTestSuite) TestCollectionsComponentCollectionsSupportedCollectionExists() {
cName := "test"
sName := "_default"
cfgMgr := new(mockConfigManager)
cfgMgr.On("AddConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
dispatcher := new(mockDispatcher)
dispatcher.On("SetPostCompleteErrorHandler", mock.AnythingOfType("gocbcore.postCompleteErrorHandler")).Return()
dispatcher.On("CollectionsEnabled").Return(true).Once()
dispatcher.On("SupportsCollections").Return(true).Once()
dispatcher.On("DispatchDirect", mock.AnythingOfType("*gocbcore.memdQRequest")).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdCollectionsGetID, req.Command)
suite.Assert().Equal([]byte(fmt.Sprintf("%s.%s", sName, cName)), req.Value)
suite.Assert().Empty(req.Key)
suite.Assert().Empty(req.CollectionName)
suite.Assert().Empty(req.ScopeName)
suite.Assert().Equal(-1, req.ReplicaIdx)
extras := make([]byte, 12)
binary.BigEndian.PutUint64(extras[0:], 1)
binary.BigEndian.PutUint32(extras[8:], 8)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{Extras: extras}}, req, nil)
})
})
dispatcher.On("RequeueDirect", mock.AnythingOfType("*gocbcore.memdQRequest"), false).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdGet, req.Command)
suite.Assert().Equal([]byte("test-key"), req.Key)
suite.Assert().Equal(cName, req.CollectionName)
suite.Assert().Equal(sName, req.ScopeName)
suite.Assert().Equal(uint32(8), req.CollectionID)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{Value: []byte("test")}}, req, nil)
})
})
cidMgr := newCollectionIDManager(collectionIDProps{
DefaultRetryStrategy: &failFastRetryStrategy{},
MaxQueueSize: 100},
dispatcher,
newTracerComponent(&noopTracer{}, "", true, &noopMeter{}),
cfgMgr,
)
cidMgr.configSeen = 1
waitCh := make(chan error, 1)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
waitCh <- err
}
// This request should get queued as the manager hasn't seen a config.
op, err := cidMgr.Dispatch(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: []byte("test-key"),
Value: nil,
},
CollectionName: cName,
ScopeName: sName,
Callback: handler,
RootTraceContext: noopSpanContext{},
})
suite.Require().Nil(err, err)
suite.Assert().NotNil(op)
select {
case <-time.After(1 * time.Second):
suite.T().Fatalf("Timed out waiting for callback to be called")
case err := <-waitCh:
suite.Assert().Nil(err, err)
}
cfgMgr.AssertExpectations(suite.T())
dispatcher.AssertExpectations(suite.T())
}
// This test is for the scenario when a request is made whilst collections support is known
// and collections are enabled, the server does support them, and the collection doesn't exist initially but then
// comes online.
// We should see the SDK send a request to get the collection ID for the collection name and
// then the user's request be failed.
func (suite *UnitTestSuite) TestCollectionsComponentCollectionsSupportedCollectionComesOnline() {
cName := "test"
sName := "_default"
cfgMgr := new(mockConfigManager)
cfgMgr.On("AddConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
dispatcher := new(mockDispatcher)
dispatcher.On("SetPostCompleteErrorHandler", mock.AnythingOfType("gocbcore.postCompleteErrorHandler")).Return()
dispatcher.On("CollectionsEnabled").Return(true).Once()
dispatcher.On("SupportsCollections").Return(true).Once()
dispatcher.On("DispatchDirect", mock.AnythingOfType("*gocbcore.memdQRequest")).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdCollectionsGetID, req.Command)
suite.Assert().Equal([]byte(fmt.Sprintf("%s.%s", sName, cName)), req.Value)
suite.Assert().Empty(req.Key)
suite.Assert().Empty(req.CollectionName)
suite.Assert().Empty(req.ScopeName)
suite.Assert().Equal(-1, req.ReplicaIdx)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{}}, req, errCollectionNotFound)
})
}).Once()
dispatcher.On("DispatchDirect", mock.AnythingOfType("*gocbcore.memdQRequest")).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdCollectionsGetID, req.Command)
suite.Assert().Equal([]byte(fmt.Sprintf("%s.%s", sName, cName)), req.Value)
suite.Assert().Empty(req.Key)
suite.Assert().Empty(req.CollectionName)
suite.Assert().Empty(req.ScopeName)
suite.Assert().Equal(-1, req.ReplicaIdx)
extras := make([]byte, 12)
binary.BigEndian.PutUint64(extras[0:], 1)
binary.BigEndian.PutUint32(extras[8:], 8)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{Extras: extras}}, req, nil)
})
}).Once()
dispatcher.On("RequeueDirect", mock.AnythingOfType("*gocbcore.memdQRequest"), false).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdGet, req.Command)
suite.Assert().Equal([]byte("test-key"), req.Key)
suite.Assert().Equal(cName, req.CollectionName)
suite.Assert().Equal(sName, req.ScopeName)
suite.Assert().Equal(uint32(8), req.CollectionID)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{Value: []byte("test")}}, req, nil)
})
}).Once()
cidMgr := newCollectionIDManager(collectionIDProps{
DefaultRetryStrategy: &failFastRetryStrategy{},
MaxQueueSize: 100},
dispatcher,
newTracerComponent(&noopTracer{}, "", true, &noopMeter{}),
cfgMgr,
)
cidMgr.configSeen = 1
waitCh := make(chan error, 1)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
waitCh <- err
}
// This request should get queued as the manager hasn't seen a config.
op, err := cidMgr.Dispatch(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: []byte("test-key"),
Value: nil,
},
CollectionName: cName,
ScopeName: sName,
Callback: handler,
RootTraceContext: noopSpanContext{},
})
suite.Require().Nil(err, err)
suite.Assert().NotNil(op)
// Requeueing on cid unknown is done in a go routine
select {
case <-time.After(1 * time.Second):
suite.T().Fatalf("Timed out waiting for callback to be called")
case err := <-waitCh:
suite.Assert().Nil(err, err)
}
cfgMgr.AssertExpectations(suite.T())
dispatcher.AssertExpectations(suite.T())
}
// This test extends TestCollectionsComponentCollectionsSupportedCollectionExists to add a second
// request which should be dispatched with no extra calls.
func (suite *UnitTestSuite) TestCollectionsComponentCollectionsSupportedCollectionUpdate() {
cName := "test"
sName := "_default"
cfgMgr := new(mockConfigManager)
cfgMgr.On("AddConfigWatcher", mock.AnythingOfType("*gocbcore.collectionsComponent")).Return()
initialDispatchesDoneCh := make(chan struct{})
dispatcher := new(mockDispatcher)
dispatcher.On("SetPostCompleteErrorHandler", mock.AnythingOfType("gocbcore.postCompleteErrorHandler")).Return()
dispatcher.On("CollectionsEnabled").Return(true).Times(3)
dispatcher.On("SupportsCollections").Return(true).Times(3)
// The first request to dispatch getting the cid.
dispatcher.On("DispatchDirect", mock.AnythingOfType("*gocbcore.memdQRequest")).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdCollectionsGetID, req.Command)
suite.Assert().Equal([]byte(fmt.Sprintf("%s.%s", sName, cName)), req.Value)
suite.Assert().Empty(req.Key)
suite.Assert().Empty(req.CollectionName)
suite.Assert().Empty(req.ScopeName)
suite.Assert().Equal(-1, req.ReplicaIdx)
extras := make([]byte, 12)
binary.BigEndian.PutUint64(extras[0:], 1)
binary.BigEndian.PutUint32(extras[8:], 8)
go func() {
<-initialDispatchesDoneCh
req.Callback(&memdQResponse{Packet: &memd.Packet{Extras: extras}}, req, nil)
}()
}).Once()
// The second request should be queued due to cid being pending so it should get requeued.
dispatcher.On("RequeueDirect", mock.AnythingOfType("*gocbcore.memdQRequest"), false).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdGet, req.Command)
suite.Assert().Equal([]byte("test-key"), req.Key)
suite.Assert().Equal(cName, req.CollectionName)
suite.Assert().Equal(sName, req.ScopeName)
suite.Assert().Equal(uint32(8), req.CollectionID)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{Value: []byte("test")}}, req, nil)
})
}).Twice()
// The third request should go straight through to Dispatch.
dispatcher.On("DispatchDirect", mock.AnythingOfType("*gocbcore.memdQRequest")).Return(&memdQRequest{}, nil).
Run(func(args mock.Arguments) {
req := args[0].(*memdQRequest)
suite.Assert().Equal(memd.CmdMagicReq, req.Magic)
suite.Assert().Equal(memd.CmdGet, req.Command)
suite.Assert().Equal([]byte("test-key"), req.Key)
suite.Assert().Equal(cName, req.CollectionName)
suite.Assert().Equal(sName, req.ScopeName)
suite.Assert().Equal(uint32(8), req.CollectionID)
time.AfterFunc(time.Millisecond, func() {
req.Callback(&memdQResponse{Packet: &memd.Packet{Value: []byte("test")}}, req, nil)
})
}).Once()
cidMgr := newCollectionIDManager(collectionIDProps{
DefaultRetryStrategy: &failFastRetryStrategy{},
MaxQueueSize: 100},
dispatcher,
newTracerComponent(&noopTracer{}, "", true, &noopMeter{}),
cfgMgr,
)
cidMgr.configSeen = 1
// This request should get queued as the manager hasn't seen a config.
op, err := cidMgr.Dispatch(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: []byte("test-key"),
Value: nil,
},
CollectionName: cName,
ScopeName: sName,
Callback: func(resp *memdQResponse, req *memdQRequest, err error) {
},
RootTraceContext: noopSpanContext{},
})
suite.Require().Nil(err, err)
suite.Assert().NotNil(op)
// This request should get queued because the cid is pending, it will then be requeued.
waitCh := make(chan error)
op, err = cidMgr.Dispatch(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: []byte("test-key"),
Value: nil,
},
CollectionName: cName,
ScopeName: sName,
Callback: func(resp *memdQResponse, req *memdQRequest, err error) {
waitCh <- err
},
RootTraceContext: noopSpanContext{},
})
suite.Require().Nil(err, err)
suite.Assert().NotNil(op)
close(initialDispatchesDoneCh)
select {
case <-time.After(1 * time.Second):
suite.T().Fatalf("Timed out waiting for callback to be called")
case err := <-waitCh:
suite.Assert().Nil(err, err)
}
waitCh = make(chan error)
op, err = cidMgr.Dispatch(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: []byte("test-key"),
Value: nil,
},
CollectionName: cName,
ScopeName: sName,
Callback: func(resp *memdQResponse, req *memdQRequest, err error) {
waitCh <- err
},
RootTraceContext: noopSpanContext{},
})
suite.Require().Nil(err, err)
suite.Assert().NotNil(op)
select {
case <-time.After(1 * time.Second):
suite.T().Fatalf("Timed out waiting for callback to be called")
case err := <-waitCh:
suite.Assert().Nil(err, err)
}
cfgMgr.AssertExpectations(suite.T())
dispatcher.AssertExpectations(suite.T())
}
gocbcore-10.2.3/commonflags.go 0000664 0000000 0000000 00000005053 14417540156 0016226 0 ustar 00root root 0000000 0000000 package gocbcore
const (
// Legacy flag format for JSON data.
lfJSON = 0
// Common flags mask
cfMask = 0xFF000000
// Common flags mask for data format
cfFmtMask = 0x0F000000
// Common flags mask for compression mode.
cfCmprMask = 0xE0000000
// Common flag format for sdk-private data.
cfFmtPrivate = 1 << 24 // nolint: deadcode,varcheck,unused
// Common flag format for JSON data.
cfFmtJSON = 2 << 24
// Common flag format for binary data.
cfFmtBinary = 3 << 24
// Common flag format for string data.
cfFmtString = 4 << 24
// Common flags compression for disabled compression.
cfCmprNone = 0 << 29
)
// DataType represents the type of data for a value
type DataType uint32
// CompressionType indicates the type of compression for a value
type CompressionType uint32
const (
// UnknownType indicates the values type is unknown.
UnknownType = DataType(0)
// JSONType indicates the value is JSON data.
JSONType = DataType(1)
// BinaryType indicates the value is binary data.
BinaryType = DataType(2)
// StringType indicates the value is string data.
StringType = DataType(3)
)
const (
// UnknownCompression indicates that the compression type is unknown.
UnknownCompression = CompressionType(0)
// NoCompression indicates that no compression is being used.
NoCompression = CompressionType(1)
)
// EncodeCommonFlags encodes a data type and compression type into a flags
// value using the common flags specification.
func EncodeCommonFlags(valueType DataType, compression CompressionType) uint32 {
var flags uint32
switch valueType {
case JSONType:
flags |= cfFmtJSON
case BinaryType:
flags |= cfFmtBinary
case StringType:
flags |= cfFmtString
case UnknownType:
// flags |= ?
}
switch compression {
case NoCompression:
// flags |= 0
case UnknownCompression:
// flags |= ?
}
return flags
}
// DecodeCommonFlags decodes a flags value into a data type and compression type
// using the common flags specification.
func DecodeCommonFlags(flags uint32) (DataType, CompressionType) {
// Check for legacy flags
if flags&cfMask == 0 {
// Legacy Flags
if flags == lfJSON {
// Legacy JSON
flags = cfFmtJSON
} else {
return UnknownType, UnknownCompression
}
}
valueType := UnknownType
compression := UnknownCompression
if flags&cfFmtMask == cfFmtBinary {
valueType = BinaryType
} else if flags&cfFmtMask == cfFmtString {
valueType = StringType
} else if flags&cfFmtMask == cfFmtJSON {
valueType = JSONType
}
if flags&cfCmprMask == cfCmprNone {
compression = NoCompression
}
return valueType, compression
}
gocbcore-10.2.3/config.go 0000664 0000000 0000000 00000035750 14417540156 0015175 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"fmt"
"net"
"strings"
)
// A Node is a computer in a cluster running the couchbase software.
type cfgNode struct {
ClusterCompatibility int `json:"clusterCompatibility"`
ClusterMembership string `json:"clusterMembership"`
CouchAPIBase string `json:"couchApiBase"`
Hostname string `json:"hostname"`
InterestingStats map[string]float64 `json:"interestingStats,omitempty"`
MCDMemoryAllocated float64 `json:"mcdMemoryAllocated"`
MCDMemoryReserved float64 `json:"mcdMemoryReserved"`
MemoryFree float64 `json:"memoryFree"`
MemoryTotal float64 `json:"memoryTotal"`
OS string `json:"os"`
Ports map[string]int `json:"ports"`
Status string `json:"status"`
Uptime int `json:"uptime,string"`
Version string `json:"version"`
ThisNode bool `json:"thisNode,omitempty"`
}
type cfgNodeServices struct {
Kv uint16 `json:"kv"`
Capi uint16 `json:"capi"`
Mgmt uint16 `json:"mgmt"`
N1ql uint16 `json:"n1ql"`
Fts uint16 `json:"fts"`
Cbas uint16 `json:"cbas"`
Eventing uint16 `json:"eventingAdminPort"`
GSI uint16 `json:"indexHttp"`
Backup uint16 `json:"backupAPI"`
KvSsl uint16 `json:"kvSSL"`
CapiSsl uint16 `json:"capiSSL"`
MgmtSsl uint16 `json:"mgmtSSL"`
N1qlSsl uint16 `json:"n1qlSSL"`
FtsSsl uint16 `json:"ftsSSL"`
CbasSsl uint16 `json:"cbasSSL"`
EventingSsl uint16 `json:"eventingSSL"`
GSISsl uint16 `json:"indexHttps"`
BackupSsl uint16 `json:"backupAPIHTTPS"`
}
type cfgNodeAltAddress struct {
Ports *cfgNodeServices `json:"ports,omitempty"`
Hostname string `json:"hostname"`
}
type cfgNodeExt struct {
Services cfgNodeServices `json:"services"`
Hostname string `json:"hostname"`
ThisNode bool `json:"thisNode"`
AltAddresses map[string]cfgNodeAltAddress `json:"alternateAddresses"`
}
// VBucketServerMap is the a mapping of vbuckets to nodes.
type cfgVBucketServerMap struct {
HashAlgorithm string `json:"hashAlgorithm"`
NumReplicas int `json:"numReplicas"`
ServerList []string `json:"serverList"`
VBucketMap [][]int `json:"vBucketMap"`
}
// Bucket is the primary entry point for most data operations.
type cfgBucket struct {
Rev int64 `json:"rev"`
RevEpoch int64 `json:"revEpoch"`
SourceHostname string
Capabilities []string `json:"bucketCapabilities"`
CapabilitiesVersion string `json:"bucketCapabilitiesVer"`
Name string `json:"name"`
NodeLocator string `json:"nodeLocator"`
URI string `json:"uri"`
StreamingURI string `json:"streamingUri"`
UUID string `json:"uuid"`
DDocs struct {
URI string `json:"uri"`
} `json:"ddocs,omitempty"`
// These are used for JSON IO, but isn't used for processing
// since it needs to be swapped out safely.
VBucketServerMap cfgVBucketServerMap `json:"vBucketServerMap"`
Nodes []cfgNode `json:"nodes"`
NodesExt []cfgNodeExt `json:"nodesExt,omitempty"`
ClusterCapabilitiesVer []int `json:"clusterCapabilitiesVer,omitempty"`
ClusterCapabilities map[string][]string `json:"clusterCapabilities,omitempty"`
}
// BuildRouteConfig builds a new route config from this config.
// overwriteSeedNode indicates that we should set the hostname for a node to the cfg.SourceHostname when the config has
// been sourced from that node.
func (cfg *cfgBucket) BuildRouteConfig(useSsl bool, networkType string, firstConnect bool, overwriteSeedNode bool) *routeConfig {
var (
kvServerList = routeEndpoints{}
capiEpList = routeEndpoints{}
mgmtEpList = routeEndpoints{}
n1qlEpList = routeEndpoints{}
ftsEpList = routeEndpoints{}
cbasEpList = routeEndpoints{}
eventingEpList = routeEndpoints{}
gsiEpList = routeEndpoints{}
backupEpList = routeEndpoints{}
bktType bucketType
)
switch cfg.NodeLocator {
case "ketama":
bktType = bktTypeMemcached
case "vbucket":
bktType = bktTypeCouchbase
default:
if cfg.UUID == "" {
bktType = bktTypeNone
} else {
logDebugf("Invalid nodeLocator %s", cfg.NodeLocator)
bktType = bktTypeInvalid
}
}
if cfg.NodesExt != nil {
lenNodes := len(cfg.Nodes)
for i, node := range cfg.NodesExt {
hostname := node.Hostname
ports := node.Services
if networkType != "default" {
if altAddr, ok := node.AltAddresses[networkType]; ok {
hostname = altAddr.Hostname
if altAddr.Ports != nil {
ports = *altAddr.Ports
}
} else {
if !firstConnect {
logDebugf("Invalid config network type %s", networkType)
}
continue
}
}
isSeedNode := node.ThisNode || len(node.Hostname) == 0
if isSeedNode && overwriteSeedNode {
logSchedf("Seed node detected and set to overwrite, setting hostname to %s", cfg.SourceHostname)
hostname = cfg.SourceHostname
} else {
hostname = getHostname(hostname, cfg.SourceHostname)
}
endpoints := endpointsFromPorts(ports, hostname, isSeedNode)
if endpoints.kvServer.Address != "" {
if bktType > bktTypeInvalid && i >= lenNodes {
logDebugf("KV node present in nodesext but not in nodes for %s", endpoints.kvServer.Address)
} else {
kvServerList.NonSSLEndpoints = append(kvServerList.NonSSLEndpoints, endpoints.kvServer)
}
}
if endpoints.capiEp.Address != "" {
capiEpList.NonSSLEndpoints = append(capiEpList.NonSSLEndpoints, endpoints.capiEp)
}
if endpoints.mgmtEp.Address != "" {
mgmtEpList.NonSSLEndpoints = append(mgmtEpList.NonSSLEndpoints, endpoints.mgmtEp)
}
if endpoints.n1qlEp.Address != "" {
n1qlEpList.NonSSLEndpoints = append(n1qlEpList.NonSSLEndpoints, endpoints.n1qlEp)
}
if endpoints.ftsEp.Address != "" {
ftsEpList.NonSSLEndpoints = append(ftsEpList.NonSSLEndpoints, endpoints.ftsEp)
}
if endpoints.cbasEp.Address != "" {
cbasEpList.NonSSLEndpoints = append(cbasEpList.NonSSLEndpoints, endpoints.cbasEp)
}
if endpoints.eventingEp.Address != "" {
eventingEpList.NonSSLEndpoints = append(eventingEpList.NonSSLEndpoints, endpoints.eventingEp)
}
if endpoints.gsiEp.Address != "" {
gsiEpList.NonSSLEndpoints = append(gsiEpList.NonSSLEndpoints, endpoints.gsiEp)
}
if endpoints.backupEp.Address != "" {
backupEpList.NonSSLEndpoints = append(backupEpList.NonSSLEndpoints, endpoints.backupEp)
}
if endpoints.kvServerSSL.Address != "" {
if bktType > bktTypeInvalid && i >= lenNodes {
logDebugf("KV node present in nodesext but not in nodes for %s", endpoints.kvServerSSL)
} else {
kvServerList.SSLEndpoints = append(kvServerList.SSLEndpoints, endpoints.kvServerSSL)
}
}
if endpoints.capiEpSSL.Address != "" {
capiEpList.SSLEndpoints = append(capiEpList.SSLEndpoints, endpoints.capiEpSSL)
}
if endpoints.mgmtEpSSL.Address != "" {
mgmtEpList.SSLEndpoints = append(mgmtEpList.SSLEndpoints, endpoints.mgmtEpSSL)
}
if endpoints.n1qlEpSSL.Address != "" {
n1qlEpList.SSLEndpoints = append(n1qlEpList.SSLEndpoints, endpoints.n1qlEpSSL)
}
if endpoints.ftsEpSSL.Address != "" {
ftsEpList.SSLEndpoints = append(ftsEpList.SSLEndpoints, endpoints.ftsEpSSL)
}
if endpoints.cbasEpSSL.Address != "" {
cbasEpList.SSLEndpoints = append(cbasEpList.SSLEndpoints, endpoints.cbasEpSSL)
}
if endpoints.eventingEpSSL.Address != "" {
eventingEpList.SSLEndpoints = append(eventingEpList.SSLEndpoints, endpoints.eventingEpSSL)
}
if endpoints.gsiEpSSL.Address != "" {
gsiEpList.SSLEndpoints = append(gsiEpList.SSLEndpoints, endpoints.gsiEpSSL)
}
if endpoints.backupEpSSL.Address != "" {
backupEpList.SSLEndpoints = append(backupEpList.SSLEndpoints, endpoints.backupEpSSL)
}
}
} else {
if useSsl {
logErrorf("Received config without nodesExt while SSL is enabled. Generating invalid config.")
return &routeConfig{}
}
if bktType == bktTypeCouchbase {
for _, s := range cfg.VBucketServerMap.ServerList {
kvServerList.NonSSLEndpoints = append(kvServerList.NonSSLEndpoints, routeEndpoint{
Address: s,
})
}
}
for _, node := range cfg.Nodes {
if node.CouchAPIBase != "" {
// Slice off the UUID as Go's HTTP client cannot handle being passed URL-Encoded path values.
capiEp := strings.SplitN(node.CouchAPIBase, "%2B", 2)[0]
capiEpList.NonSSLEndpoints = append(capiEpList.NonSSLEndpoints, routeEndpoint{
Address: capiEp,
})
}
if node.Hostname != "" {
mgmtEpList.NonSSLEndpoints = append(mgmtEpList.NonSSLEndpoints, routeEndpoint{
Address: fmt.Sprintf("http://%s", node.Hostname),
})
}
if bktType == bktTypeMemcached {
// Get the data port. No VBucketServerMap.
host, err := hostFromHostPort(node.Hostname)
if err != nil {
logErrorf("Encountered invalid memcached host/port string. Ignoring node.")
continue
}
curKvHost := fmt.Sprintf("%s:%d", host, node.Ports["direct"])
kvServerList.NonSSLEndpoints = append(kvServerList.NonSSLEndpoints, routeEndpoint{
Address: curKvHost,
})
}
}
}
rc := &routeConfig{
revID: cfg.Rev,
revEpoch: cfg.RevEpoch,
uuid: cfg.UUID,
name: cfg.Name,
kvServerList: kvServerList,
capiEpList: capiEpList,
mgmtEpList: mgmtEpList,
n1qlEpList: n1qlEpList,
ftsEpList: ftsEpList,
cbasEpList: cbasEpList,
eventingEpList: eventingEpList,
gsiEpList: gsiEpList,
backupEpList: backupEpList,
bktType: bktType,
clusterCapabilities: cfg.ClusterCapabilities,
clusterCapabilitiesVer: cfg.ClusterCapabilitiesVer,
bucketCapabilities: cfg.Capabilities,
bucketCapabilitiesVer: cfg.CapabilitiesVersion,
}
if bktType == bktTypeCouchbase {
vbMap := cfg.VBucketServerMap.VBucketMap
numReplicas := cfg.VBucketServerMap.NumReplicas
rc.vbMap = newVbucketMap(vbMap, numReplicas)
} else if bktType == bktTypeMemcached {
var endpoints []routeEndpoint
if useSsl {
endpoints = kvServerList.SSLEndpoints
} else {
endpoints = kvServerList.NonSSLEndpoints
}
rc.ketamaMap = newKetamaContinuum(endpoints)
}
return rc
}
type serverEps struct {
kvServerSSL routeEndpoint
capiEpSSL routeEndpoint
mgmtEpSSL routeEndpoint
n1qlEpSSL routeEndpoint
ftsEpSSL routeEndpoint
cbasEpSSL routeEndpoint
eventingEpSSL routeEndpoint
gsiEpSSL routeEndpoint
backupEpSSL routeEndpoint
kvServer routeEndpoint
capiEp routeEndpoint
mgmtEp routeEndpoint
n1qlEp routeEndpoint
ftsEp routeEndpoint
cbasEp routeEndpoint
eventingEp routeEndpoint
gsiEp routeEndpoint
backupEp routeEndpoint
}
func getHostname(hostname, sourceHostname string) string {
// Hostname blank means to use the same one as was connected to
if hostname == "" {
// Note that the SourceHostname will already be IPv6 wrapped
hostname = sourceHostname
} else {
// We need to detect an IPv6 address here and wrap it in the appropriate
// [] block to indicate its IPv6 for the rest of the system.
if strings.Contains(hostname, ":") {
hostname = "[" + hostname + "]"
}
}
return hostname
}
func endpointsFromPorts(ports cfgNodeServices, hostname string, isSeedNode bool) *serverEps {
lists := &serverEps{}
if ports.KvSsl > 0 {
lists.kvServerSSL = routeEndpoint{
Address: fmt.Sprintf("couchbases://%s:%d", hostname, ports.KvSsl),
IsSeedNode: isSeedNode,
}
}
if ports.CapiSsl > 0 {
lists.capiEpSSL = routeEndpoint{
Address: fmt.Sprintf("https://%s:%d", hostname, ports.CapiSsl),
IsSeedNode: isSeedNode,
}
}
if ports.MgmtSsl > 0 {
lists.mgmtEpSSL = routeEndpoint{
Address: fmt.Sprintf("https://%s:%d", hostname, ports.MgmtSsl),
IsSeedNode: isSeedNode,
}
}
if ports.N1qlSsl > 0 {
lists.n1qlEpSSL = routeEndpoint{
Address: fmt.Sprintf("https://%s:%d", hostname, ports.N1qlSsl),
IsSeedNode: isSeedNode,
}
}
if ports.FtsSsl > 0 {
lists.ftsEpSSL = routeEndpoint{
Address: fmt.Sprintf("https://%s:%d", hostname, ports.FtsSsl),
IsSeedNode: isSeedNode,
}
}
if ports.CbasSsl > 0 {
lists.cbasEpSSL = routeEndpoint{
Address: fmt.Sprintf("https://%s:%d", hostname, ports.CbasSsl),
IsSeedNode: isSeedNode,
}
}
if ports.EventingSsl > 0 {
lists.eventingEpSSL = routeEndpoint{
Address: fmt.Sprintf("https://%s:%d", hostname, ports.EventingSsl),
IsSeedNode: isSeedNode,
}
}
if ports.GSISsl > 0 {
lists.gsiEpSSL = routeEndpoint{
Address: fmt.Sprintf("https://%s:%d", hostname, ports.GSISsl),
IsSeedNode: isSeedNode,
}
}
if ports.BackupSsl > 0 {
lists.backupEpSSL = routeEndpoint{
Address: fmt.Sprintf("https://%s:%d", hostname, ports.BackupSsl),
IsSeedNode: isSeedNode,
}
}
if ports.Kv > 0 {
lists.kvServer = routeEndpoint{
Address: fmt.Sprintf("couchbase://%s:%d", hostname, ports.Kv),
IsSeedNode: isSeedNode,
}
}
if ports.Capi > 0 {
lists.capiEp = routeEndpoint{
Address: fmt.Sprintf("http://%s:%d", hostname, ports.Capi),
IsSeedNode: isSeedNode,
}
}
if ports.Mgmt > 0 {
lists.mgmtEp = routeEndpoint{
Address: fmt.Sprintf("http://%s:%d", hostname, ports.Mgmt),
IsSeedNode: isSeedNode,
}
}
if ports.N1ql > 0 {
lists.n1qlEp = routeEndpoint{
Address: fmt.Sprintf("http://%s:%d", hostname, ports.N1ql),
IsSeedNode: isSeedNode,
}
}
if ports.Fts > 0 {
lists.ftsEp = routeEndpoint{
Address: fmt.Sprintf("http://%s:%d", hostname, ports.Fts),
IsSeedNode: isSeedNode,
}
}
if ports.Cbas > 0 {
lists.cbasEp = routeEndpoint{
Address: fmt.Sprintf("http://%s:%d", hostname, ports.Cbas),
IsSeedNode: isSeedNode,
}
}
if ports.Eventing > 0 {
lists.eventingEp = routeEndpoint{
Address: fmt.Sprintf("http://%s:%d", hostname, ports.Eventing),
IsSeedNode: isSeedNode,
}
}
if ports.GSI > 0 {
lists.gsiEp = routeEndpoint{
Address: fmt.Sprintf("http://%s:%d", hostname, ports.GSI),
IsSeedNode: isSeedNode,
}
}
if ports.Backup > 0 {
lists.backupEp = routeEndpoint{
Address: fmt.Sprintf("http://%s:%d", hostname, ports.Backup),
IsSeedNode: isSeedNode,
}
}
return lists
}
func hostFromHostPort(hostport string) (string, error) {
host, _, err := net.SplitHostPort(hostport)
if err != nil {
return "", err
}
// If this is an IPv6 address, we need to rewrap it in []
if strings.Contains(host, ":") {
return "[" + host + "]", nil
}
return host, nil
}
func parseConfig(config []byte, srcHost string) (*cfgBucket, error) {
configStr := strings.Replace(string(config), "$HOST", srcHost, -1)
bk := new(cfgBucket)
err := json.Unmarshal([]byte(configStr), bk)
if err != nil {
return nil, err
}
bk.SourceHostname = srcHost
return bk, nil
}
gocbcore-10.2.3/configmanagement_component.go 0000664 0000000 0000000 00000015325 14417540156 0021310 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"sync"
)
type configManagementComponent struct {
useSSL bool
networkType string
noTLSSeedNode bool
currentConfig *routeConfig
configLock sync.Mutex
cfgChangeWatchers []routeConfigWatcher
watchersLock sync.Mutex
srcServers []routeEndpoint
seenConfig bool
}
type configManagerProperties struct {
UseTLS bool
NoTLSSeedNode bool
NetworkType string
SrcMemdAddrs []routeEndpoint
SrcHTTPAddrs []routeEndpoint
}
type routeConfigWatcher interface {
OnNewRouteConfig(cfg *routeConfig)
}
type configManager interface {
AddConfigWatcher(watcher routeConfigWatcher)
RemoveConfigWatcher(watcher routeConfigWatcher)
}
func newConfigManager(props configManagerProperties) *configManagementComponent {
return &configManagementComponent{
useSSL: props.UseTLS,
noTLSSeedNode: props.NoTLSSeedNode,
networkType: props.NetworkType,
srcServers: append(props.SrcMemdAddrs, props.SrcHTTPAddrs...),
currentConfig: &routeConfig{
revID: -1,
},
}
}
func (cm *configManagementComponent) UseTLS(use bool) {
cm.configLock.Lock()
cm.useSSL = use
cm.configLock.Unlock()
}
func (cm *configManagementComponent) TLSEnabled() bool {
cm.configLock.Lock()
useSSL := cm.useSSL
cm.configLock.Unlock()
return useSSL
}
func (cm *configManagementComponent) OnNewConfig(cfg *cfgBucket) {
var routeCfg *routeConfig
cm.configLock.Lock()
if cm.seenConfig {
routeCfg = cfg.BuildRouteConfig(cm.useSSL, cm.networkType, false, cm.noTLSSeedNode)
} else {
routeCfg = cm.buildFirstRouteConfig(cfg, cm.useSSL)
logDebugf("Using network type %s for connections", cm.networkType)
}
if !routeCfg.IsValid() {
cm.configLock.Unlock()
logDebugf("Routing data is not valid, skipping update: \n%s", routeCfg.DebugString())
return
}
// There's something wrong with this route config so don't send it to the watchers.
if !cm.canUpdateRouteConfig(routeCfg) {
cm.configLock.Unlock()
return
}
cm.currentConfig = routeCfg
cm.seenConfig = true
cm.configLock.Unlock()
logDebugf("Sending out mux routing data (update)...")
logDebugf("New Routing Data:\n%s", routeCfg.DebugString())
// We can end up deadlocking if we iterate whilst in the lock and a watcher decides to remove itself.
cm.watchersLock.Lock()
watchers := make([]routeConfigWatcher, len(cm.cfgChangeWatchers))
copy(watchers, cm.cfgChangeWatchers)
cm.watchersLock.Unlock()
for _, watcher := range watchers {
watcher.OnNewRouteConfig(routeCfg)
}
}
func (cm *configManagementComponent) Watchers() []routeConfigWatcher {
cm.watchersLock.Lock()
watchers := make([]routeConfigWatcher, len(cm.cfgChangeWatchers))
copy(watchers, cm.cfgChangeWatchers)
cm.watchersLock.Unlock()
return watchers
}
func (cm *configManagementComponent) ResetConfig() {
cm.configLock.Lock()
cm.currentConfig = &routeConfig{
revID: -1,
}
cm.configLock.Unlock()
}
func (cm *configManagementComponent) AddConfigWatcher(watcher routeConfigWatcher) {
cm.watchersLock.Lock()
cm.cfgChangeWatchers = append(cm.cfgChangeWatchers, watcher)
cm.watchersLock.Unlock()
}
func (cm *configManagementComponent) RemoveConfigWatcher(watcher routeConfigWatcher) {
var idx int
var found bool
cm.watchersLock.Lock()
for i, w := range cm.cfgChangeWatchers {
if w == watcher {
idx = i
found = true
break
}
}
if !found {
cm.watchersLock.Unlock()
return
}
if idx == len(cm.cfgChangeWatchers) {
cm.cfgChangeWatchers = cm.cfgChangeWatchers[:idx]
} else {
cm.cfgChangeWatchers = append(cm.cfgChangeWatchers[:idx], cm.cfgChangeWatchers[idx+1:]...)
}
cm.watchersLock.Unlock()
}
// We should never be receiving concurrent updates and nothing should be accessing
// our internal route config so we shouldn't need to lock here.
func (cm *configManagementComponent) canUpdateRouteConfig(cfg *routeConfig) bool {
oldCfg := cm.currentConfig
// Check some basic things to ensure consistency!
// If oldCfg name was empty and the new cfg isn't then we're moving from cluster to bucket connection.
if oldCfg.revID > -1 && (oldCfg.name != "" && cfg.name != "") {
if (cfg.vbMap == nil) != (oldCfg.vbMap == nil) {
logErrorf("Received a configuration with a different number of vbuckets %s-%s. Ignoring.", oldCfg.name, cfg.name)
return false
}
if cfg.vbMap != nil && cfg.vbMap.NumVbuckets() != oldCfg.vbMap.NumVbuckets() {
logErrorf("Received a configuration with a different number of vbuckets %s-%s. Ignoring.", oldCfg.name, cfg.name)
return false
}
}
// Check that the new config data is newer than the current one, in the case where we've done a select bucket
// against an existing connection then the revisions could be the same. In that case the configuration still
// needs to be applied.
// In the case where the rev epochs are the same then we need to compare rev IDs. If the new config epoch is lower
// than the old one then we ignore it, if it's newer then we apply the new config.
if cfg.bktType != oldCfg.bktType {
logDebugf("Configuration data changed bucket type, switching.")
} else if !cfg.IsNewerThan(oldCfg) {
return false
}
return true
}
func (cm *configManagementComponent) buildFirstRouteConfig(config *cfgBucket, useSSL bool) *routeConfig {
if cm.networkType != "" && cm.networkType != "auto" {
return config.BuildRouteConfig(useSSL, cm.networkType, true, cm.noTLSSeedNode)
}
defaultRouteConfig := config.BuildRouteConfig(useSSL, "default", true, cm.noTLSSeedNode)
var kvServerList []routeEndpoint
var mgmtEpList []routeEndpoint
if useSSL {
kvServerList = defaultRouteConfig.kvServerList.SSLEndpoints
mgmtEpList = defaultRouteConfig.mgmtEpList.SSLEndpoints
} else {
kvServerList = defaultRouteConfig.kvServerList.NonSSLEndpoints
mgmtEpList = defaultRouteConfig.mgmtEpList.NonSSLEndpoints
}
// Iterate over all the source servers and check if any addresses match as default or external network types
for _, srcServer := range cm.srcServers {
// First we check if the source server is from the defaults list
srcInDefaultConfig := false
for _, endpoint := range kvServerList {
if trimSchemePrefix(endpoint.Address) == srcServer.Address {
srcInDefaultConfig = true
}
}
for _, endpoint := range mgmtEpList {
if endpoint == srcServer {
srcInDefaultConfig = true
}
}
if srcInDefaultConfig {
cm.networkType = "default"
return defaultRouteConfig
}
}
// Next lets see if we have an external config, if so, default to that
externalRouteCfg := config.BuildRouteConfig(useSSL, "external", true, cm.noTLSSeedNode)
if externalRouteCfg.IsValid() {
cm.networkType = "external"
return externalRouteCfg
}
// If all else fails, default to the implicit default config
cm.networkType = "default"
return defaultRouteConfig
}
func (cm *configManagementComponent) NetworkType() string {
return cm.networkType
}
gocbcore-10.2.3/configmanagement_component_test.go 0000664 0000000 0000000 00000006304 14417540156 0022344 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"testing"
)
type testRouteWatcher struct {
receivedConfig *routeConfig
}
func (trw *testRouteWatcher) OnNewRouteConfig(cfg *routeConfig) {
trw.receivedConfig = cfg
}
func (suite *UnitTestSuite) TestConfigComponentRevEpoch() {
data, err := suite.LoadRawTestDataset("bucket_config_with_rev_epoch")
suite.Require().Nil(err)
var cfg *cfgBucket
suite.Require().Nil(json.Unmarshal(data, &cfg))
type tCase struct {
name string
prevRevID int64
prevRevEpoch int64
newRevID int64
newRevEpoch int64
expectUpdate bool
}
testCases := []tCase{
{
name: "no_epoch_newer_rev",
prevRevID: 1,
prevRevEpoch: 0,
newRevID: 2,
newRevEpoch: 0,
expectUpdate: true,
},
{
name: "no_epoch_same_rev",
prevRevID: 1,
prevRevEpoch: 0,
newRevID: 1,
newRevEpoch: 0,
expectUpdate: false,
},
{
name: "no_epoch_older_rev",
prevRevID: 2,
prevRevEpoch: 0,
newRevID: 1,
newRevEpoch: 0,
expectUpdate: false,
},
{
name: "same_epoch_newer_rev",
prevRevID: 1,
prevRevEpoch: 5,
newRevID: 2,
newRevEpoch: 5,
expectUpdate: true,
},
{
name: "same_epoch_same_rev",
prevRevID: 1,
prevRevEpoch: 5,
newRevID: 1,
newRevEpoch: 5,
expectUpdate: false,
},
{
name: "same_epoch_older_rev",
prevRevID: 2,
prevRevEpoch: 5,
newRevID: 1,
newRevEpoch: 5,
expectUpdate: false,
},
{
name: "newer_epoch_newer_rev",
prevRevID: 1,
prevRevEpoch: 5,
newRevID: 2,
newRevEpoch: 6,
expectUpdate: true,
},
{
name: "newer_epoch_same_rev",
prevRevID: 1,
prevRevEpoch: 5,
newRevID: 1,
newRevEpoch: 6,
expectUpdate: true,
},
{
name: "newer_epoch_older_rev",
prevRevID: 2,
prevRevEpoch: 5,
newRevID: 1,
newRevEpoch: 6,
expectUpdate: true,
},
{
name: "older_epoch_newer_rev",
prevRevID: 1,
prevRevEpoch: 5,
newRevID: 2,
newRevEpoch: 4,
expectUpdate: false,
},
{
name: "older_epoch_same_rev",
prevRevID: 1,
prevRevEpoch: 5,
newRevID: 1,
newRevEpoch: 4,
expectUpdate: false,
},
{
name: "older_epoch_older_rev",
prevRevID: 2,
prevRevEpoch: 5,
newRevID: 1,
newRevEpoch: 4,
expectUpdate: false,
},
}
for _, tCase := range testCases {
suite.T().Run(tCase.name, func(te *testing.T) {
oldCfg := *cfg
oldCfg.Rev = tCase.prevRevID
oldCfg.RevEpoch = tCase.prevRevEpoch
watcher := &testRouteWatcher{}
cmpt := configManagementComponent{
useSSL: false,
networkType: "default",
cfgChangeWatchers: []routeConfigWatcher{watcher},
currentConfig: oldCfg.BuildRouteConfig(false, "default", false, false),
}
newCfg := *cfg
newCfg.Rev = tCase.newRevID
newCfg.RevEpoch = tCase.newRevEpoch
cmpt.OnNewConfig(&newCfg)
if tCase.expectUpdate {
if watcher.receivedConfig == nil {
te.Fatalf("Watcher didn't receive config")
}
} else {
if watcher.receivedConfig != nil {
te.Fatalf("Watcher did receive config")
}
}
})
}
}
gocbcore-10.2.3/configsnapshot.go 0000664 0000000 0000000 00000004650 14417540156 0016750 0 ustar 00root root 0000000 0000000 package gocbcore
// ConfigSnapshot is a snapshot of the underlying configuration currently in use.
type ConfigSnapshot struct {
state *kvMuxState
}
// RevID returns the config revision for this snapshot.
func (pi ConfigSnapshot) RevID() int64 {
return pi.state.RevID()
}
// KeyToVbucket translates a particular key to its assigned vbucket.
func (pi ConfigSnapshot) KeyToVbucket(key []byte) (uint16, error) {
if pi.state.VBMap() == nil {
return 0, errUnsupportedOperation
}
return pi.state.VBMap().VbucketByKey(key), nil
}
// KeyToServer translates a particular key to its assigned server index.
func (pi ConfigSnapshot) KeyToServer(key []byte, replicaIdx uint32) (int, error) {
if pi.state.VBMap() != nil {
serverIdx, err := pi.state.VBMap().NodeByKey(key, replicaIdx)
if err != nil {
return 0, err
}
return serverIdx, nil
}
if pi.state.KetamaMap() != nil {
serverIdx, err := pi.state.KetamaMap().NodeByKey(key)
if err != nil {
return 0, err
}
return serverIdx, nil
}
return 0, errCliInternalError
}
// VbucketToServer returns the server index for a particular vbucket.
func (pi ConfigSnapshot) VbucketToServer(vbID uint16, replicaIdx uint32) (int, error) {
if pi.state.VBMap() == nil {
return 0, errUnsupportedOperation
}
serverIdx, err := pi.state.VBMap().NodeByVbucket(vbID, replicaIdx)
if err != nil {
return 0, err
}
return serverIdx, nil
}
// VbucketsOnServer returns the list of VBuckets for a server.
func (pi ConfigSnapshot) VbucketsOnServer(index int) ([]uint16, error) {
if pi.state.VBMap() == nil {
return nil, errUnsupportedOperation
}
return pi.state.VBMap().VbucketsOnServer(index)
}
// NumVbuckets returns the number of VBuckets configured on the
// connected cluster.
func (pi ConfigSnapshot) NumVbuckets() (int, error) {
if pi.state.VBMap() == nil {
return 0, errUnsupportedOperation
}
return pi.state.VBMap().NumVbuckets(), nil
}
// NumReplicas returns the number of replicas configured on the
// connected cluster.
func (pi ConfigSnapshot) NumReplicas() (int, error) {
if pi.state.VBMap() == nil {
return 0, errUnsupportedOperation
}
return pi.state.VBMap().NumReplicas(), nil
}
// NumServers returns the number of servers accessible for K/V.
func (pi ConfigSnapshot) NumServers() (int, error) {
return pi.state.NumPipelines(), nil
}
// BucketUUID returns the UUID of the bucket we are connected to.
func (pi ConfigSnapshot) BucketUUID() string {
return pi.state.UUID()
}
gocbcore-10.2.3/connstr/ 0000775 0000000 0000000 00000000000 14417540156 0015055 5 ustar 00root root 0000000 0000000 gocbcore-10.2.3/connstr/README.md 0000664 0000000 0000000 00000002037 14417540156 0016336 0 ustar 00root root 0000000 0000000 # Couchbase Connection Strings for Go
This library allows you to parse and resolve Couchbase Connection Strings in Go.
This is used by the Couchbase Go SDK, as well as various tools throughout the
Couchbase infrastructure.
## Using the Library
To parse a connection string, simply call `Parse` with your connection string.
You will receive a `ConnSpec` structure representing the connection string`:
```go
type Address struct {
Host string
Port int
}
type ConnSpec struct {
Scheme string
Addresses []Address
Bucket string
Options map[string][]string
}
```
One you have a parsed connection string, you can also use our resolver to take
the `ConnSpec` and resolve any DNS SRV records as well as generate a list of
endpoints for the Couchbase server. You will receive a `ResolvedConnSpec`
structure in return:
```go
type ResolvedConnSpec struct {
UseSsl bool
MemdHosts []Address
HttpHosts []Address
Bucket string
Options map[string][]string
}
```
## License
Copyright 2020 Couchbase Inc.
Licensed under the Apache License, Version 2.0.
gocbcore-10.2.3/connstr/connstr.go 0000664 0000000 0000000 00000020302 14417540156 0017067 0 ustar 00root root 0000000 0000000 package connstr
import (
"errors"
"fmt"
"net"
"net/url"
"regexp"
"strconv"
"strings"
)
const (
// DefaultHttpPort is the default HTTP port to use to connect to Couchbase Server.
DefaultHttpPort = 8091
// DefaultSslHttpPort is the default HTTPS port to use to connect to Couchbase Server.
DefaultSslHttpPort = 18091
// DefaultMemdPort is the default memd port to use to connect to Couchbase Server.
DefaultMemdPort = 11210
// DefaultSslMemdPort is the default memd SSL port to use to connect to Couchbase Server.
DefaultSslMemdPort = 11207
)
const (
couchbaseScheme = iota + 1
httpScheme
nsServerScheme
)
func hostIsIpAddress(host string) bool {
if strings.HasPrefix(host, "[") {
// This is an IPv6 address
return true
}
if net.ParseIP(host) != nil {
// This is an IPv4 address
return true
}
return false
}
// Address represents a host:port pair.
type Address struct {
Host string
Port int
}
// ConnSpec describes a connection specification.
type ConnSpec struct {
Scheme string
Addresses []Address
Bucket string
Options map[string][]string
}
func (spec ConnSpec) srvRecord() (string, string, string, bool) {
// Only `couchbase`-type schemes allow SRV records
if spec.Scheme != "couchbase" && spec.Scheme != "couchbases" {
return "", "", "", false
}
// Must have only a single host, with no port specified
if len(spec.Addresses) != 1 || spec.Addresses[0].Port != -1 {
return "", "", "", false
}
if hostIsIpAddress(spec.Addresses[0].Host) {
return "", "", "", false
}
return spec.Scheme, "tcp", spec.Addresses[0].Host, true
}
// SrvRecordName returns the record name for the ConnSpec.
func (spec ConnSpec) SrvRecordName() (recordName string) {
scheme, proto, host, isValid := spec.srvRecord()
if !isValid {
return ""
}
return fmt.Sprintf("_%s._%s.%s", scheme, proto, host)
}
// GetOption returns the specified option value for the ConnSpec.
func (spec ConnSpec) GetOption(name string) []string {
if opt, ok := spec.Options[name]; ok {
return opt
}
return nil
}
// GetOptionString returns the specified option value for the ConnSpec.
func (spec ConnSpec) GetOptionString(name string) string {
opts := spec.GetOption(name)
if len(opts) > 0 {
return opts[0]
}
return ""
}
// Parse parses the connection string into a ConnSpec.
func Parse(connStr string) (out ConnSpec, err error) {
partMatcher := regexp.MustCompile(`((.*):\/\/)?(([^\/?:]*)(:([^\/?:@]*))?@)?([^\/?]*)(\/([^\?]*))?(\?(.*))?`)
hostMatcher := regexp.MustCompile(`((\[[^\]]+\]+)|([^;\,\:]+))(:([0-9]*))?(;\,)?`)
parts := partMatcher.FindStringSubmatch(connStr)
var onlyAllowSingleHost bool
if parts[2] != "" {
out.Scheme = parts[2]
switch out.Scheme {
case "couchbase":
case "couchbases":
case "http":
case "ns_server":
onlyAllowSingleHost = true
case "ns_servers":
onlyAllowSingleHost = true
default:
err = errors.New("bad scheme")
return
}
}
if parts[7] != "" {
hosts := hostMatcher.FindAllStringSubmatch(parts[7], -1)
if len(hosts) > 1 && onlyAllowSingleHost {
err = errors.New("ns_server scheme can only be used with a single host")
return
}
for _, hostInfo := range hosts {
address := Address{
Host: hostInfo[1],
Port: -1,
}
if hostInfo[5] != "" {
address.Port, err = strconv.Atoi(hostInfo[5])
if err != nil {
return
}
}
out.Addresses = append(out.Addresses, address)
}
}
if parts[9] != "" {
out.Bucket, err = url.QueryUnescape(parts[9])
if err != nil {
return
}
}
if parts[11] != "" {
out.Options, err = url.ParseQuery(parts[11])
if err != nil {
return
}
}
return
}
func (spec ConnSpec) String() string {
var out string
if spec.Scheme != "" {
out += fmt.Sprintf("%s://", spec.Scheme)
}
for i, address := range spec.Addresses {
if i > 0 {
out += ","
}
if address.Port >= 0 {
out += fmt.Sprintf("%s:%d", address.Host, address.Port)
} else {
out += address.Host
}
}
if spec.Bucket != "" {
out += "/"
out += spec.Bucket
}
urlOptions := url.Values(spec.Options)
if len(urlOptions) > 0 {
out += "?" + urlOptions.Encode()
}
return out
}
// ResolvedConnSpec is the result of resolving a ConnSpec.
type ResolvedConnSpec struct {
UseSsl bool
MemdHosts []Address
HttpHosts []Address
NSServerHost *Address
Bucket string
Options map[string][]string
SrvRecord *SrvRecord
}
// SrvRecord contains the information about the srv record used to extract addresses.
type SrvRecord struct {
Proto string
Scheme string
Host string
}
// Resolve parses a ConnSpec into a ResolvedConnSpec.
func Resolve(connSpec ConnSpec) (out ResolvedConnSpec, err error) {
defaultPort := 0
hasExplicitScheme := false
var scheme int
useSsl := false
switch connSpec.Scheme {
case "couchbase":
defaultPort = DefaultMemdPort
hasExplicitScheme = true
scheme = couchbaseScheme
useSsl = false
case "couchbases":
defaultPort = DefaultSslMemdPort
hasExplicitScheme = true
scheme = couchbaseScheme
useSsl = true
case "http":
defaultPort = DefaultHttpPort
hasExplicitScheme = true
scheme = httpScheme
useSsl = false
case "ns_server":
defaultPort = DefaultHttpPort
hasExplicitScheme = true
scheme = nsServerScheme
useSsl = true
case "":
defaultPort = DefaultHttpPort
hasExplicitScheme = false
scheme = httpScheme
useSsl = false
default:
err = errors.New("bad scheme")
return
}
var srvRecords []*net.SRV
srvScheme, srvProto, srvHost, srvIsValid := connSpec.srvRecord()
if srvIsValid {
_, addrs, err := net.LookupSRV(srvScheme, srvProto, srvHost)
if err == nil && len(addrs) > 0 {
srvRecords = addrs
}
}
if srvRecords != nil {
for _, srv := range srvRecords {
out.MemdHosts = append(out.MemdHosts, Address{
Host: strings.TrimSuffix(srv.Target, "."),
Port: int(srv.Port),
})
}
out.SrvRecord = &SrvRecord{
Host: srvHost,
Proto: srvProto,
Scheme: srvScheme,
}
} else if len(connSpec.Addresses) == 0 {
if scheme == nsServerScheme {
out.NSServerHost = &Address{
Host: "127.0.0.1",
Port: DefaultHttpPort,
}
} else {
if useSsl {
out.MemdHosts = append(out.MemdHosts, Address{
Host: "127.0.0.1",
Port: DefaultSslMemdPort,
})
out.HttpHosts = append(out.HttpHosts, Address{
Host: "127.0.0.1",
Port: DefaultSslHttpPort,
})
} else {
out.MemdHosts = append(out.MemdHosts, Address{
Host: "127.0.0.1",
Port: DefaultMemdPort,
})
out.HttpHosts = append(out.HttpHosts, Address{
Host: "127.0.0.1",
Port: DefaultHttpPort,
})
}
}
} else {
for _, address := range connSpec.Addresses {
hasExplicitPort := address.Port > 0
if !hasExplicitScheme && hasExplicitPort && address.Port != defaultPort {
err = errors.New("ambiguous port without scheme")
return
}
if hasExplicitScheme && scheme == couchbaseScheme && address.Port == DefaultHttpPort {
err = errors.New("couchbase://host:8091 not supported for couchbase:// scheme. Use couchbase://host")
return
}
if address.Port <= 0 || address.Port == defaultPort || address.Port == DefaultHttpPort {
if scheme == nsServerScheme {
out.NSServerHost = &Address{
Host: address.Host,
Port: DefaultHttpPort,
}
} else {
if useSsl {
out.MemdHosts = append(out.MemdHosts, Address{
Host: address.Host,
Port: DefaultSslMemdPort,
})
out.HttpHosts = append(out.HttpHosts, Address{
Host: address.Host,
Port: DefaultSslHttpPort,
})
} else {
out.MemdHosts = append(out.MemdHosts, Address{
Host: address.Host,
Port: DefaultMemdPort,
})
out.HttpHosts = append(out.HttpHosts, Address{
Host: address.Host,
Port: DefaultHttpPort,
})
}
}
} else {
switch scheme {
case couchbaseScheme:
out.MemdHosts = append(out.MemdHosts, Address{
Host: address.Host,
Port: address.Port,
})
case httpScheme:
out.HttpHosts = append(out.HttpHosts, Address{
Host: address.Host,
Port: address.Port,
})
case nsServerScheme:
out.NSServerHost = &Address{
Host: address.Host,
Port: address.Port,
}
}
}
}
}
out.UseSsl = useSsl
out.Bucket = connSpec.Bucket
out.Options = connSpec.Options
return
}
gocbcore-10.2.3/connstr/connstr_test.go 0000664 0000000 0000000 00000024570 14417540156 0020141 0 ustar 00root root 0000000 0000000 package connstr
import (
"testing"
)
func parseOrDie(t *testing.T, connStr string) ConnSpec {
cs, err := Parse(connStr)
if err != nil {
t.Fatalf("Failed to parse %s: %v", connStr, err)
}
return cs
}
func resolveOrDie(t *testing.T, connSpec ConnSpec) ResolvedConnSpec {
rcs, err := Resolve(connSpec)
if err != nil {
t.Fatalf("Failed to resolve %s: %v", connSpec, err)
}
return rcs
}
func checkSpec(t *testing.T, connStr string, expectedSpec ConnSpec, expectMemdHosts []Address, expectHttpHosts []Address, expectNSServerHost *Address, useSsl bool, checkHosts bool, checkStr bool) {
cs := parseOrDie(t, connStr)
if checkStr && cs.String() != connStr {
t.Fatalf("ConnStr round-trip should match. %s != %s", cs.String(), connStr)
}
if cs.Scheme != expectedSpec.Scheme {
t.Fatalf("Parsed incorrect scheme")
}
if len(cs.Addresses) != len(expectedSpec.Addresses) {
t.Fatalf("Some addresses were not parsed")
}
for i, csAddr := range cs.Addresses {
expectedAddr := expectedSpec.Addresses[i]
if csAddr.Host != expectedAddr.Host {
t.Fatalf("Parsed incorrect host. %s != %s", csAddr.Host, expectedAddr.Host)
}
if csAddr.Port != expectedAddr.Port {
t.Fatalf("Parsed incorrect port. %d != %d", csAddr.Port, expectedAddr.Port)
}
}
if cs.Bucket != expectedSpec.Bucket {
t.Fatalf("Parsed incorrect bucket. %s != %s", cs.Bucket, expectedSpec.Bucket)
}
if len(cs.Options) != len(expectedSpec.Options) {
t.Fatalf("Some options were not parsed")
}
for key, opts := range cs.Options {
expectedOpts := expectedSpec.Options[key]
if len(opts) != len(expectedOpts) {
t.Fatalf("Some option values were not parsed")
}
for i, opt := range opts {
expectedOpt := expectedOpts[i]
if opt != expectedOpt {
t.Fatalf("Parsed incorrect option value. %s != %s", opt, expectedOpt)
}
}
}
rcs := resolveOrDie(t, cs)
if rcs.UseSsl != useSsl {
t.Fatalf("Did not correctly mark SSL")
}
if checkHosts {
if len(rcs.MemdHosts) != len(expectMemdHosts) {
t.Fatalf("Some memd hosts were missing")
}
for i, host := range rcs.MemdHosts {
expectHost := expectMemdHosts[i]
if host.Host != expectHost.Host {
t.Fatalf("Resolved incorrect memd host. %s != %s", host.Host, expectHost.Host)
}
if host.Port != expectHost.Port {
t.Fatalf("Resolved incorrect memd port. %d != %d", host.Port, expectHost.Port)
}
}
if len(rcs.HttpHosts) != len(expectHttpHosts) {
t.Fatalf("Some http hosts were missing")
}
for i, host := range rcs.HttpHosts {
expectHost := expectHttpHosts[i]
if host.Host != expectHost.Host {
t.Fatalf("Resolved incorrect http host. %s != %s", host.Host, expectHost.Host)
}
if host.Port != expectHost.Port {
t.Fatalf("Resolved incorrect http port. %d != %d", host.Port, expectHost.Port)
}
}
if (rcs.NSServerHost == nil) != (expectNSServerHost == nil) {
t.Fatalf("Some ns_server hosts were missing")
}
if expectNSServerHost != nil {
if rcs.NSServerHost.Host != expectNSServerHost.Host {
t.Fatalf("Resolved incorrect ns_server host. %s != %s", rcs.NSServerHost.Host, expectNSServerHost.Host)
}
if rcs.NSServerHost.Port != expectNSServerHost.Port {
t.Fatalf("Resolved incorrect ns_server port. %d != %d", rcs.NSServerHost.Port, expectNSServerHost.Port)
}
}
}
}
func TestParseBasic(t *testing.T) {
checkSpec(t, "couchbase://1.2.3.4", ConnSpec{
Scheme: "couchbase",
Addresses: []Address{
{"1.2.3.4", -1}},
}, []Address{
{"1.2.3.4", DefaultMemdPort},
}, []Address{
{"1.2.3.4", DefaultHttpPort},
}, nil, false, true, true)
checkSpec(t, "couchbase://[2001:4860:4860::8888]", ConnSpec{
Scheme: "couchbase",
Addresses: []Address{
{"[2001:4860:4860::8888]", -1}},
}, []Address{
{"[2001:4860:4860::8888]", DefaultMemdPort},
}, []Address{
{"[2001:4860:4860::8888]", DefaultHttpPort},
}, nil, false, true, true)
_, err := Parse("blah://foo.com")
if err == nil {
t.Fatalf("Expected error for bad scheme")
}
checkSpec(t, "couchbase://", ConnSpec{
Scheme: "couchbase",
}, []Address{
{"127.0.0.1", DefaultMemdPort},
}, []Address{
{"127.0.0.1", DefaultHttpPort},
}, nil, false, true, true)
checkSpec(t, "couchbase://?", ConnSpec{
Scheme: "couchbase",
}, []Address{
{"127.0.0.1", DefaultMemdPort},
}, []Address{
{"127.0.0.1", DefaultHttpPort},
}, nil, false, true, false)
checkSpec(t, "1.2.3.4", ConnSpec{
Addresses: []Address{
{"1.2.3.4", -1},
},
}, []Address{
{"1.2.3.4", DefaultMemdPort},
}, []Address{
{"1.2.3.4", DefaultHttpPort},
}, nil, false, true, true)
checkSpec(t, "[2001:4860:4860::8888]", ConnSpec{
Addresses: []Address{
{"[2001:4860:4860::8888]", -1}},
}, []Address{
{"[2001:4860:4860::8888]", DefaultMemdPort},
}, []Address{
{"[2001:4860:4860::8888]", DefaultHttpPort},
}, nil, false, true, true)
checkSpec(t, "1.2.3.4:8091", ConnSpec{
Addresses: []Address{
{"1.2.3.4", 8091},
},
}, []Address{
{"1.2.3.4", DefaultMemdPort},
}, []Address{
{"1.2.3.4", DefaultHttpPort},
}, nil, false, true, true)
cs := parseOrDie(t, "1.2.3.4:999")
_, err = Resolve(cs)
if err == nil {
t.Fatalf("Expected error with non-default port without scheme")
}
}
func TestParseHosts(t *testing.T) {
checkSpec(t, "couchbase://foo.com,bar.com,baz.com", ConnSpec{
Scheme: "couchbase",
Addresses: []Address{
{"foo.com", -1},
{"bar.com", -1},
{"baz.com", -1},
},
}, []Address{
{"foo.com", DefaultMemdPort},
{"bar.com", DefaultMemdPort},
{"baz.com", DefaultMemdPort},
}, []Address{
{"foo.com", DefaultHttpPort},
{"bar.com", DefaultHttpPort},
{"baz.com", DefaultHttpPort},
}, nil, false, true, true)
checkSpec(t, "couchbase://[2001:4860:4860::8822],[2001:4860:4860::8833]:888", ConnSpec{
Scheme: "couchbase",
Addresses: []Address{
{"[2001:4860:4860::8822]", -1},
{"[2001:4860:4860::8833]", 888},
},
}, []Address{
{"[2001:4860:4860::8822]", DefaultMemdPort},
{"[2001:4860:4860::8833]", 888},
}, []Address{
{"[2001:4860:4860::8822]", DefaultHttpPort},
}, nil, false, true, true)
// Parse using legacy format
cs := parseOrDie(t, "couchbase://foo.com:8091")
_, err := Resolve(cs)
if err == nil {
t.Fatalf("Expected error for couchbase://XXX:8091")
}
checkSpec(t, "couchbase://foo.com:4444", ConnSpec{
Scheme: "couchbase",
Addresses: []Address{
{"foo.com", 4444},
},
}, []Address{
{"foo.com", 4444},
}, nil, nil, false, true, true)
checkSpec(t, "couchbases://foo.com:4444", ConnSpec{
Scheme: "couchbases",
Addresses: []Address{
{"foo.com", 4444},
},
}, []Address{
{"foo.com", 4444},
}, []Address{}, nil, true, true, true)
checkSpec(t, "couchbases://", ConnSpec{
Scheme: "couchbases",
}, []Address{
{"127.0.0.1", DefaultSslMemdPort},
}, []Address{
{"127.0.0.1", DefaultSslHttpPort},
}, nil, true, true, true)
checkSpec(t, "couchbase://foo.com,bar.com:4444", ConnSpec{
Scheme: "couchbase",
Addresses: []Address{
{"foo.com", -1},
{"bar.com", 4444},
},
}, []Address{
{"foo.com", DefaultMemdPort},
{"bar.com", 4444},
}, []Address{
{"foo.com", DefaultHttpPort},
}, nil, false, true, true)
checkSpec(t, "couchbase://foo.com;bar.com;baz.com", ConnSpec{
Scheme: "couchbase",
Addresses: []Address{
{"foo.com", -1},
{"bar.com", -1},
{"baz.com", -1},
},
}, []Address{
{"foo.com", DefaultMemdPort},
{"bar.com", DefaultMemdPort},
{"baz.com", DefaultMemdPort},
}, []Address{
{"foo.com", DefaultHttpPort},
{"bar.com", DefaultHttpPort},
{"baz.com", DefaultHttpPort},
}, nil, false, true, false)
}
func TestParseBucket(t *testing.T) {
checkSpec(t, "couchbase://foo.com/user", ConnSpec{
Scheme: "couchbase",
Addresses: []Address{
{"foo.com", -1},
},
Bucket: "user",
}, nil, nil, nil, false, false, false)
checkSpec(t, "couchbase://foo.com/user/", ConnSpec{
Scheme: "couchbase",
Addresses: []Address{
{"foo.com", -1},
},
Bucket: "user/",
}, nil, nil, nil, false, false, false)
checkSpec(t, "couchbase:///default", ConnSpec{
Scheme: "couchbase",
Bucket: "default",
}, nil, nil, nil, false, false, false)
checkSpec(t, "couchbase:///default", ConnSpec{
Scheme: "couchbase",
Bucket: "default",
}, nil, nil, nil, false, false, false)
checkSpec(t, "couchbase:///default", ConnSpec{
Scheme: "couchbase",
Bucket: "default",
}, nil, nil, nil, false, false, false)
checkSpec(t, "couchbase:///default?", ConnSpec{
Scheme: "couchbase",
Bucket: "default",
}, nil, nil, nil, false, false, false)
checkSpec(t, "couchbase:///%2FUsers%2F?", ConnSpec{
Scheme: "couchbase",
Bucket: "/Users/",
}, nil, nil, nil, false, false, false)
}
func TestOptionsPassthrough(t *testing.T) {
checkSpec(t, "couchbase:///?foo=bar", ConnSpec{
Scheme: "couchbase",
Options: map[string][]string{
"foo": {"bar"},
},
}, nil, nil, nil, false, false, false)
checkSpec(t, "couchbase://?foo=bar", ConnSpec{
Scheme: "couchbase",
Options: map[string][]string{
"foo": {"bar"},
},
}, nil, nil, nil, false, false, true)
checkSpec(t, "couchbase://?foo=fooval&bar=barval", ConnSpec{
Scheme: "couchbase",
Options: map[string][]string{
"foo": {"fooval"},
"bar": {"barval"},
},
}, nil, nil, nil, false, false, false)
checkSpec(t, "couchbase://?foo=fooval&bar=barval&", ConnSpec{
Scheme: "couchbase",
Options: map[string][]string{
"foo": {"fooval"},
"bar": {"barval"},
},
}, nil, nil, nil, false, false, false)
checkSpec(t, "couchbase://?foo=val1&foo=val2&", ConnSpec{
Scheme: "couchbase",
Options: map[string][]string{
"foo": {"val1", "val2"},
},
}, nil, nil, nil, false, false, false)
}
func TestParseNSServer(t *testing.T) {
checkSpec(t, "ns_server://1.2.3.4", ConnSpec{
Scheme: "ns_server",
Addresses: []Address{
{"1.2.3.4", -1}},
}, []Address{}, []Address{}, &Address{
"1.2.3.4", DefaultHttpPort,
}, true, true, true)
checkSpec(t, "ns_server://", ConnSpec{
Scheme: "ns_server",
}, []Address{}, []Address{}, &Address{
"127.0.0.1", DefaultHttpPort,
}, true, true, true)
checkSpec(t, "ns_server://1.2.3.4:1234", ConnSpec{
Scheme: "ns_server",
Addresses: []Address{
{"1.2.3.4", 1234}},
}, []Address{}, []Address{}, &Address{
"1.2.3.4", 1234,
}, true, true, true)
_, err := Parse("ns_server://1.2.3.4,1.2.3.5")
if err == nil {
t.Fatalf("Parse should fail for more than 1 address with ns_server scheme")
}
checkSpec(t, "ns_server://1.2.3.4:8091", ConnSpec{
Scheme: "ns_server",
Addresses: []Address{
{"1.2.3.4", 8091},
},
}, []Address{}, []Address{}, &Address{
"1.2.3.4", DefaultHttpPort,
}, true, true, true)
}
gocbcore-10.2.3/constants.go 0000664 0000000 0000000 00000010577 14417540156 0015744 0 ustar 00root root 0000000 0000000 package gocbcore
const (
goCbCoreVersionStr = "v10.2.3"
)
type bucketType int
const (
bktTypeNone = -1
bktTypeInvalid bucketType = 0
bktTypeCouchbase = iota
bktTypeMemcached = iota
)
// ServiceType specifies a particular Couchbase service type.
type ServiceType int
const (
// MemdService represents a memcached service.
MemdService = ServiceType(1)
// MgmtService represents a management service (typically ns_server).
MgmtService = ServiceType(2)
// CapiService represents a CouchAPI service (typically for views).
CapiService = ServiceType(3)
// N1qlService represents a N1QL service (typically for query).
N1qlService = ServiceType(4)
// FtsService represents a full-text-search service.
FtsService = ServiceType(5)
// CbasService represents an analytics service.
CbasService = ServiceType(6)
// EventingService represents the eventing service.
EventingService = ServiceType(7)
// GSIService represents the indexing service.
GSIService = ServiceType(8)
// BackupService represents the backup service.
BackupService = ServiceType(9)
)
// DcpAgentPriority specifies the priority level for a dcp stream
type DcpAgentPriority uint8
const (
// DcpAgentPriorityLow sets the priority for the dcp stream to low
DcpAgentPriorityLow = DcpAgentPriority(0)
// DcpAgentPriorityMed sets the priority for the dcp stream to medium
DcpAgentPriorityMed = DcpAgentPriority(1)
// DcpAgentPriorityHigh sets the priority for the dcp stream to high
DcpAgentPriorityHigh = DcpAgentPriority(2)
)
type BucketCapability uint32
const (
BucketCapabilityDurableWrites BucketCapability = 0x00
BucketCapabilityCreateAsDeleted BucketCapability = 0x01
BucketCapabilityReplaceBodyWithXattr BucketCapability = 0x02
BucketCapabilityRangeScan BucketCapability = 0x03
)
type BucketCapabilityStatus uint32
const (
BucketCapabilityStatusUnknown BucketCapabilityStatus = 0x00
BucketCapabilityStatusSupported BucketCapabilityStatus = 0x01
BucketCapabilityStatusUnsupported BucketCapabilityStatus = 0x02
)
// ClusterCapability represents a capability that the cluster supports
type ClusterCapability uint32
const (
// ClusterCapabilityEnhancedPreparedStatements represents that the cluster supports enhanced prepared statements.
ClusterCapabilityEnhancedPreparedStatements = ClusterCapability(0x01)
)
// DCPBackfillOrder represents the order in which vBuckets will be backfilled by the cluster.
type DCPBackfillOrder uint8
const (
// DCPBackfillOrderRoundRobin means that all the requested vBuckets will be backfilled together where each vBucket
// has some data backfilled before moving on to the next. This is the default behaviour.
DCPBackfillOrderRoundRobin DCPBackfillOrder = iota + 1
// DCPBackfillOrderSequential means that all the data for the first vBucket will be streamed before advancing onto
// the next vBucket.
DCPBackfillOrderSequential
)
const (
spanNameDispatchToServer = "dispatch_to_server"
spanAttribDBSystemKey = "db.system"
spanAttribDBSystemValue = "couchbase"
spanAttribNetTransportKey = "net.transport"
spanAttribNetTransportValue = "IP.TCP"
spanAttribOperationIDKey = "db.couchbase.operation_id"
spanAttribLocalIDKey = "db.couchbase.local_id"
spanAttribNetHostNameKey = "net.host.name"
spanAttribNetHostPortKey = "net.host.port"
spanAttribNetPeerNameKey = "net.peer.name"
spanAttribNetPeerPortKey = "net.peer.port"
spanAttribServerDurationKey = "db.couchbase.server_duration"
spanAttribNumRetries = "db.couchbase.retries"
)
const (
metricAttribServiceKey = "db.couchbase.service"
metricAttribOperationKey = "db.operation"
meterNameCBOperations = "db.couchbase.operations"
metricValueServiceKeyValue = "kv"
metricValueServiceQueryValue = "n1ql"
metricValueServiceSearchValue = "fts"
metricValueServiceAnalyticsValue = "cbas"
metricValueServiceViewsValue = "capi"
metricValueServiceHTTPValue = "http"
)
type SpanStatus string
const (
SpanStatusOK SpanStatus = "Ok"
SpanStatusError SpanStatus = "Error"
)
type statusClass uint8
const (
statusClassOK statusClass = iota
statusClassError
)
var crc32cMacro = []byte("\"${Mutation.value_crc32c}\"")
var revidMacro = []byte("\"${$document.revid}\"")
var exptimeMacro = []byte("\"${$document.exptime}\"")
var casMacro = []byte("\"${$document.CAS}\"")
var hlcMacro = "$vbucket.HLC"
gocbcore-10.2.3/crud.go 0000664 0000000 0000000 00000001000 14417540156 0014642 0 ustar 00root root 0000000 0000000 package gocbcore
// Cas represents a unique revision of a document. This can be used
// to perform optimistic locking.
type Cas uint64
// VbUUID represents a unique identifier for a particular vbucket history.
type VbUUID uint64
// SeqNo is a sequential mutation number indicating the order and precise
// position of a write that has occurred.
type SeqNo uint64
// MutationToken represents a particular mutation within the cluster.
type MutationToken struct {
VbID uint16
VbUUID VbUUID
SeqNo SeqNo
}
gocbcore-10.2.3/crud_dura.go 0000664 0000000 0000000 00000002621 14417540156 0015667 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
// ObserveOptions encapsulates the parameters for a ObserveEx operation.
type ObserveOptions struct {
Key []byte
ReplicaIdx int
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// ObserveVbOptions encapsulates the parameters for a ObserveVbEx operation.
type ObserveVbOptions struct {
VbID uint16
VbUUID VbUUID
ReplicaIdx int
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// ObserveResult encapsulates the result of a ObserveEx operation.
type ObserveResult struct {
KeyState memd.KeyState
Cas Cas
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// ObserveVbResult encapsulates the result of a ObserveVbEx operation.
type ObserveVbResult struct {
DidFailover bool
VbID uint16
VbUUID VbUUID
PersistSeqNo SeqNo
CurrentSeqNo SeqNo
OldVbUUID VbUUID
LastSeqNo SeqNo
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
gocbcore-10.2.3/crud_options.go 0000664 0000000 0000000 00000021225 14417540156 0016430 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
// GetOptions encapsulates the parameters for a GetEx operation.
type GetOptions struct {
Key []byte
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// GetAndTouchOptions encapsulates the parameters for a GetAndTouchEx operation.
type GetAndTouchOptions struct {
Key []byte
Expiry uint32
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// GetAndLockOptions encapsulates the parameters for a GetAndLockEx operation.
type GetAndLockOptions struct {
Key []byte
LockTime uint32
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// GetAnyReplicaOptions encapsulates the parameters for a GetAnyReplicaEx operation.
type GetAnyReplicaOptions struct {
Key []byte
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// GetOneReplicaOptions encapsulates the parameters for a GetOneReplicaEx operation.
type GetOneReplicaOptions struct {
Key []byte
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
ReplicaIdx int
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// TouchOptions encapsulates the parameters for a TouchEx operation.
type TouchOptions struct {
Key []byte
Expiry uint32
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// UnlockOptions encapsulates the parameters for a UnlockEx operation.
type UnlockOptions struct {
Key []byte
Cas Cas
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// DeleteOptions encapsulates the parameters for a DeleteEx operation.
type DeleteOptions struct {
Key []byte
CollectionName string
ScopeName string
RetryStrategy RetryStrategy
Cas Cas
DurabilityLevel memd.DurabilityLevel
DurabilityLevelTimeout time.Duration
CollectionID uint32
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// AddOptions encapsulates the parameters for a AddEx operation.
type AddOptions struct {
Key []byte
CollectionName string
ScopeName string
RetryStrategy RetryStrategy
Value []byte
Flags uint32
Datatype uint8
Expiry uint32
DurabilityLevel memd.DurabilityLevel
DurabilityLevelTimeout time.Duration
CollectionID uint32
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
type storeOptions struct {
Key []byte
CollectionName string
ScopeName string
RetryStrategy RetryStrategy
Value []byte
Flags uint32
Datatype uint8
Cas Cas
Expiry uint32
DurabilityLevel memd.DurabilityLevel
DurabilityLevelTimeout time.Duration
CollectionID uint32
Deadline time.Time
PreserveExpiry bool
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// SetOptions encapsulates the parameters for a SetEx operation.
type SetOptions struct {
Key []byte
CollectionName string
ScopeName string
RetryStrategy RetryStrategy
Value []byte
Flags uint32
Datatype uint8
Expiry uint32
DurabilityLevel memd.DurabilityLevel
DurabilityLevelTimeout time.Duration
CollectionID uint32
Deadline time.Time
PreserveExpiry bool
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// ReplaceOptions encapsulates the parameters for a ReplaceEx operation.
type ReplaceOptions struct {
Key []byte
CollectionName string
ScopeName string
RetryStrategy RetryStrategy
Value []byte
Flags uint32
Datatype uint8
Cas Cas
Expiry uint32
DurabilityLevel memd.DurabilityLevel
DurabilityLevelTimeout time.Duration
CollectionID uint32
Deadline time.Time
PreserveExpiry bool
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// AdjoinOptions encapsulates the parameters for a AppendEx or PrependEx operation.
type AdjoinOptions struct {
Key []byte
Value []byte
CollectionName string
ScopeName string
RetryStrategy RetryStrategy
Cas Cas
DurabilityLevel memd.DurabilityLevel
DurabilityLevelTimeout time.Duration
CollectionID uint32
Deadline time.Time
PreserveExpiry bool
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// CounterOptions encapsulates the parameters for a IncrementEx or DecrementEx operation.
type CounterOptions struct {
Key []byte
Delta uint64
Initial uint64
Expiry uint32
CollectionName string
ScopeName string
RetryStrategy RetryStrategy
Cas Cas
DurabilityLevel memd.DurabilityLevel
DurabilityLevelTimeout time.Duration
CollectionID uint32
Deadline time.Time
PreserveExpiry bool
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// GetRandomOptions encapsulates the parameters for a GetRandomEx operation.
type GetRandomOptions struct {
RetryStrategy RetryStrategy
Deadline time.Time
CollectionName string
ScopeName string
CollectionID uint32
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// GetMetaOptions encapsulates the parameters for a GetMetaEx operation.
type GetMetaOptions struct {
Key []byte
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// SetMetaOptions encapsulates the parameters for a SetMetaEx operation.
type SetMetaOptions struct {
Key []byte
Value []byte
Extra []byte
Datatype uint8
Options uint32
Flags uint32
Expiry uint32
Cas Cas
RevNo uint64
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// DeleteMetaOptions encapsulates the parameters for a DeleteMetaEx operation.
type DeleteMetaOptions struct {
Key []byte
Value []byte
Extra []byte
Datatype uint8
Options uint32
Flags uint32
Expiry uint32
Cas Cas
RevNo uint64
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
gocbcore-10.2.3/crud_rangescan.go 0000664 0000000 0000000 00000014047 14417540156 0016702 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/base64"
"strconv"
"time"
)
// RangeScanCreateOptions encapsulates the parameters for a RangeScanCreate operation.
// Volatile: This API is subject to change at any time.
type RangeScanCreateOptions struct {
RetryStrategy RetryStrategy
// Deadline will also be sent as a part of the payload if Snapshot is not nil.
Deadline time.Time
CollectionName string
ScopeName string
CollectionID uint32
// Note: if set then KeysOnly on RangeScanContinueOptions *must* also be set.
KeysOnly bool
Range *RangeScanCreateRangeScanConfig
Sampling *RangeScanCreateRandomSamplingConfig
Snapshot *RangeScanCreateSnapshotRequirements
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
func (opts RangeScanCreateOptions) toRequest() (*rangeScanCreateRequest, error) {
if opts.Range != nil && opts.Sampling != nil {
return nil, wrapError(errInvalidArgument, "only one of range and sampling can be set")
}
if opts.Range == nil && opts.Sampling == nil {
return nil, wrapError(errInvalidArgument, "one of range and sampling must set")
}
var collection string
if opts.CollectionID != 0 {
collection = strconv.FormatUint(uint64(opts.CollectionID), 16)
}
createReq := &rangeScanCreateRequest{
Collection: collection,
KeyOnly: opts.KeysOnly,
}
if opts.Range != nil {
if opts.Range.hasStart() && opts.Range.hasExclusiveStart() {
return nil, wrapError(errInvalidArgument, "only one of start and exclusive start within range can be set")
}
if opts.Range.hasEnd() && opts.Range.hasExclusiveEnd() {
return nil, wrapError(errInvalidArgument, "only one of end and exclusive end within range can be set")
}
if !(opts.Range.hasStart() || opts.Range.hasExclusiveStart()) {
return nil, wrapError(errInvalidArgument, "one of start and exclusive start within range must both be set")
}
if !(opts.Range.hasEnd() || opts.Range.hasExclusiveEnd()) {
return nil, wrapError(errInvalidArgument, "one of end and exclusive end within range must both be set")
}
createReq.Range = &rangeScanCreateRange{}
if len(opts.Range.Start) > 0 {
createReq.Range.Start = base64.StdEncoding.EncodeToString(opts.Range.Start)
}
if len(opts.Range.End) > 0 {
createReq.Range.End = base64.StdEncoding.EncodeToString(opts.Range.End)
}
if len(opts.Range.ExclusiveStart) > 0 {
createReq.Range.ExclusiveStart = base64.StdEncoding.EncodeToString(opts.Range.ExclusiveStart)
}
if len(opts.Range.ExclusiveEnd) > 0 {
createReq.Range.ExclusiveEnd = base64.StdEncoding.EncodeToString(opts.Range.ExclusiveEnd)
}
}
if opts.Sampling != nil {
if opts.Sampling.Samples == 0 {
return nil, wrapError(errInvalidArgument, "samples within sampling must be set")
}
createReq.Sampling = &rangeScanCreateSample{
Seed: opts.Sampling.Seed,
Samples: opts.Sampling.Samples,
}
}
if opts.Snapshot != nil {
if opts.Snapshot.VbUUID == 0 {
return nil, wrapError(errInvalidArgument, "vbuuid within snapshot must be set")
}
if opts.Snapshot.SeqNo == 0 {
return nil, wrapError(errInvalidArgument, "seqno within snapshot must be set")
}
createReq.Snapshot = &rangeScanCreateSnapshot{
VbUUID: strconv.FormatUint(uint64(opts.Snapshot.VbUUID), 10),
SeqNo: uint64(opts.Snapshot.SeqNo),
SeqNoExists: opts.Snapshot.SeqNoExists,
}
createReq.Snapshot.Timeout = uint64(time.Until(opts.Deadline).Milliseconds())
}
return createReq, nil
}
// RangeScanCreateRangeScanConfig is the configuration available for performing a range scan.
type RangeScanCreateRangeScanConfig struct {
Start []byte
End []byte
ExclusiveStart []byte
ExclusiveEnd []byte
}
func (cfg *RangeScanCreateRangeScanConfig) hasStart() bool {
return len(cfg.Start) > 0
}
func (cfg *RangeScanCreateRangeScanConfig) hasEnd() bool {
return len(cfg.End) > 0
}
func (cfg *RangeScanCreateRangeScanConfig) hasExclusiveStart() bool {
return len(cfg.ExclusiveStart) > 0
}
func (cfg *RangeScanCreateRangeScanConfig) hasExclusiveEnd() bool {
return len(cfg.ExclusiveEnd) > 0
}
// RangeScanCreateRandomSamplingConfig is the configuration available for performing a random sampling.
type RangeScanCreateRandomSamplingConfig struct {
Seed uint64
Samples uint64
}
// RangeScanCreateSnapshotRequirements is the set of requirements that the vbucket snapshot must meet in-order for
// the request to be successful.
type RangeScanCreateSnapshotRequirements struct {
VbUUID VbUUID
SeqNo SeqNo
SeqNoExists bool
}
// RangeScanCreateResult encapsulates the result of a RangeScanCreate operation.
// Volatile: This API is subject to change at any time.
type RangeScanCreateResult struct {
ScanUUUID []byte
KeysOnly bool
}
// RangeScanContinueOptions encapsulates the parameters for a RangeScanContinue operation.
// Volatile: This API is subject to change at any time.
type RangeScanContinueOptions struct {
RetryStrategy RetryStrategy
// Deadline will also be sent as a part of the payload if not zero.
Deadline time.Time
MaxCount uint32
MaxBytes uint32
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// RangeScanItem encapsulates an iterm returned during a range scan.
type RangeScanItem struct {
Value []byte
Key []byte
Flags uint32
Cas Cas
Expiry uint32
SeqNo SeqNo
Datatype uint8
}
// RangeScanContinueResult encapsulates the result of a RangeScanContinue operation.
// Volatile: This API is subject to change at any time.
type RangeScanContinueResult struct {
More bool
Complete bool
}
// RangeScanCancelOptions encapsulates the parameters for a RangeScanCancel operation.
// Volatile: This API is subject to change at any time.
type RangeScanCancelOptions struct {
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// RangeScanCancelResult encapsulates the result of a RangeScanCancel operation.
// Volatile: This API is subject to change at any time.
type RangeScanCancelResult struct{}
gocbcore-10.2.3/crud_results.go 0000664 0000000 0000000 00000010167 14417540156 0016441 0 ustar 00root root 0000000 0000000 package gocbcore
// ResourceUnitResult describes the number of compute units used by an operation.
// Internal: This should never be used and is not supported.
type ResourceUnitResult struct {
ReadUnits uint16
WriteUnits uint16
}
// GetResult encapsulates the result of a GetEx operation.
type GetResult struct {
Value []byte
Flags uint32
Datatype uint8
Cas Cas
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// GetAndTouchResult encapsulates the result of a GetAndTouchEx operation.
type GetAndTouchResult struct {
Value []byte
Flags uint32
Datatype uint8
Cas Cas
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// GetAndLockResult encapsulates the result of a GetAndLockEx operation.
type GetAndLockResult struct {
Value []byte
Flags uint32
Datatype uint8
Cas Cas
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// GetReplicaResult encapsulates the result of a GetReplica operation.
type GetReplicaResult struct {
Value []byte
Flags uint32
Datatype uint8
Cas Cas
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// TouchResult encapsulates the result of a TouchEx operation.
type TouchResult struct {
Cas Cas
MutationToken MutationToken
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// UnlockResult encapsulates the result of a UnlockEx operation.
type UnlockResult struct {
Cas Cas
MutationToken MutationToken
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// DeleteResult encapsulates the result of a DeleteEx operation.
type DeleteResult struct {
Cas Cas
MutationToken MutationToken
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// StoreResult encapsulates the result of a AddEx, SetEx or ReplaceEx operation.
type StoreResult struct {
Cas Cas
MutationToken MutationToken
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// AdjoinResult encapsulates the result of a AppendEx or PrependEx operation.
type AdjoinResult struct {
Cas Cas
MutationToken MutationToken
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// CounterResult encapsulates the result of a IncrementEx or DecrementEx operation.
type CounterResult struct {
Value uint64
Cas Cas
MutationToken MutationToken
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// GetRandomResult encapsulates the result of a GetRandomEx operation.
type GetRandomResult struct {
Key []byte
Value []byte
Flags uint32
Datatype uint8
Cas Cas
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// GetMetaResult encapsulates the result of a GetMetaEx operation.
type GetMetaResult struct {
Value []byte
Flags uint32
Cas Cas
Expiry uint32
SeqNo SeqNo
Datatype uint8
Deleted uint32
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// SetMetaResult encapsulates the result of a SetMetaEx operation.
type SetMetaResult struct {
Cas Cas
MutationToken MutationToken
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// DeleteMetaResult encapsulates the result of a DeleteMetaEx operation.
type DeleteMetaResult struct {
Cas Cas
MutationToken MutationToken
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
gocbcore-10.2.3/crud_subdoc.go 0000664 0000000 0000000 00000003527 14417540156 0016221 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
// LookupInOptions encapsulates the parameters for a LookupInEx operation.
type LookupInOptions struct {
Key []byte
Flags memd.SubdocDocFlag
Ops []SubDocOp
CollectionName string
ScopeName string
CollectionID uint32
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// MutateInOptions encapsulates the parameters for a MutateInEx operation.
type MutateInOptions struct {
Key []byte
Flags memd.SubdocDocFlag
Cas Cas
Expiry uint32
Ops []SubDocOp
CollectionName string
ScopeName string
RetryStrategy RetryStrategy
DurabilityLevel memd.DurabilityLevel
DurabilityLevelTimeout time.Duration
CollectionID uint32
Deadline time.Time
PreserveExpiry bool
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// SubDocResult encapsulates the results from a single sub-document operation.
type SubDocResult struct {
Err error
Value []byte
}
// LookupInResult encapsulates the result of a LookupInEx operation.
type LookupInResult struct {
Cas Cas
Ops []SubDocResult
// Internal: This should never be used and is not supported.
Internal struct {
IsDeleted bool
ResourceUnits *ResourceUnitResult
}
}
// MutateInResult encapsulates the result of a MutateInEx operation.
type MutateInResult struct {
Cas Cas
MutationToken MutationToken
Ops []SubDocResult
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
gocbcore-10.2.3/crudcomponent.go 0000664 0000000 0000000 00000105367 14417540156 0016612 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/binary"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
type crudComponent struct {
cidMgr *collectionsComponent
defaultRetryStrategy RetryStrategy
tracer *tracerComponent
errMapManager *errMapComponent
featureVerifier bucketCapabilityVerifier
disableDecompression bool
}
func newCRUDComponent(cidMgr *collectionsComponent, defaultRetryStrategy RetryStrategy, tracerCmpt *tracerComponent,
errMapManager *errMapComponent, featureVerifier bucketCapabilityVerifier, disableDecompression bool) *crudComponent {
return &crudComponent{
cidMgr: cidMgr,
defaultRetryStrategy: defaultRetryStrategy,
tracer: tracerCmpt,
errMapManager: errMapManager,
featureVerifier: featureVerifier,
disableDecompression: disableDecompression,
}
}
func (crud *crudComponent) Get(opts GetOptions, cb GetCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "Get", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
if len(resp.Extras) != 4 {
tracer.Finish()
cb(nil, errProtocol)
return
}
res := GetResult{}
res.Value = resp.Value
res.Flags = binary.BigEndian.Uint32(resp.Extras[0:])
res.Cas = Cas(resp.Cas)
res.Datatype = resp.Datatype
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(&res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGet,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: opts.Key,
Value: nil,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "Get", errUnambiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) GetAndTouch(opts GetAndTouchOptions, cb GetAndTouchCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "GetAndTouch", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
if len(resp.Extras) != 4 {
tracer.Finish()
cb(nil, errProtocol)
return
}
flags := binary.BigEndian.Uint32(resp.Extras[0:])
res := &GetAndTouchResult{
Value: resp.Value,
Flags: flags,
Cas: Cas(resp.Cas),
Datatype: resp.Datatype,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
extraBuf := make([]byte, 4)
binary.BigEndian.PutUint32(extraBuf[0:], opts.Expiry)
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGAT,
Datatype: 0,
Cas: 0,
Extras: extraBuf,
Key: opts.Key,
Value: nil,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "GetAndTouch", errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) GetAndLock(opts GetAndLockOptions, cb GetAndLockCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "GetAndLock", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
if len(resp.Extras) != 4 {
tracer.Finish()
cb(nil, errProtocol)
return
}
flags := binary.BigEndian.Uint32(resp.Extras[0:])
res := &GetAndLockResult{
Value: resp.Value,
Flags: flags,
Cas: Cas(resp.Cas),
Datatype: resp.Datatype,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
extraBuf := make([]byte, 4)
binary.BigEndian.PutUint32(extraBuf[0:], opts.LockTime)
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGetLocked,
Datatype: 0,
Cas: 0,
Extras: extraBuf,
Key: opts.Key,
Value: nil,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "GetAndLock", errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) GetOneReplica(opts GetOneReplicaOptions, cb GetReplicaCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "GetOneReplica", opts.TraceContext)
if opts.ReplicaIdx <= 0 {
tracer.Finish()
return nil, errInvalidReplica
}
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
if len(resp.Extras) != 4 {
tracer.Finish()
cb(nil, errProtocol)
return
}
flags := binary.BigEndian.Uint32(resp.Extras[0:])
res := &GetReplicaResult{
Value: resp.Value,
Flags: flags,
Cas: Cas(resp.Cas),
Datatype: resp.Datatype,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGetReplica,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: opts.Key,
Value: nil,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
ReplicaIdx: opts.ReplicaIdx,
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "GetOneReplica", errUnambiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) Touch(opts TouchOptions, cb TouchCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "Touch", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
mutToken := MutationToken{}
if len(resp.Extras) >= 16 {
mutToken.VbID = req.Vbucket
mutToken.VbUUID = VbUUID(binary.BigEndian.Uint64(resp.Extras[0:]))
mutToken.SeqNo = SeqNo(binary.BigEndian.Uint64(resp.Extras[8:]))
}
res := &TouchResult{
Cas: Cas(resp.Cas),
MutationToken: mutToken,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
extraBuf := make([]byte, 4)
binary.BigEndian.PutUint32(extraBuf[0:], opts.Expiry)
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdTouch,
Datatype: 0,
Cas: 0,
Extras: extraBuf,
Key: opts.Key,
Value: nil,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "Touch", errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) Unlock(opts UnlockOptions, cb UnlockCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "Unlock", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
mutToken := MutationToken{}
if len(resp.Extras) >= 16 {
mutToken.VbID = req.Vbucket
mutToken.VbUUID = VbUUID(binary.BigEndian.Uint64(resp.Extras[0:]))
mutToken.SeqNo = SeqNo(binary.BigEndian.Uint64(resp.Extras[8:]))
}
res := &UnlockResult{
Cas: Cas(resp.Cas),
MutationToken: mutToken,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdUnlockKey,
Datatype: 0,
Cas: uint64(opts.Cas),
Extras: nil,
Key: opts.Key,
Value: nil,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "Unlock", errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) Delete(opts DeleteOptions, cb DeleteCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "Delete", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
mutToken := MutationToken{}
if len(resp.Extras) >= 16 {
mutToken.VbID = req.Vbucket
mutToken.VbUUID = VbUUID(binary.BigEndian.Uint64(resp.Extras[0:]))
mutToken.SeqNo = SeqNo(binary.BigEndian.Uint64(resp.Extras[8:]))
}
res := &DeleteResult{
Cas: Cas(resp.Cas),
MutationToken: mutToken,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var duraLevelFrame *memd.DurabilityLevelFrame
var duraTimeoutFrame *memd.DurabilityTimeoutFrame
if opts.DurabilityLevel > 0 {
if crud.featureVerifier.HasBucketCapabilityStatus(BucketCapabilityDurableWrites, BucketCapabilityStatusUnsupported) {
return nil, errFeatureNotAvailable
}
duraLevelFrame = &memd.DurabilityLevelFrame{
DurabilityLevel: opts.DurabilityLevel,
}
duraTimeoutFrame = &memd.DurabilityTimeoutFrame{
DurabilityTimeout: opts.DurabilityLevelTimeout,
}
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdDelete,
Datatype: 0,
Cas: uint64(opts.Cas),
Extras: nil,
Key: opts.Key,
Value: nil,
DurabilityLevelFrame: duraLevelFrame,
DurabilityTimeoutFrame: duraTimeoutFrame,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "Delete", errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) store(opName string, opcode memd.CmdCode, opts storeOptions, cb StoreCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, opName, opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
mutToken := MutationToken{}
if len(resp.Extras) >= 16 {
mutToken.VbID = req.Vbucket
mutToken.VbUUID = VbUUID(binary.BigEndian.Uint64(resp.Extras[0:]))
mutToken.SeqNo = SeqNo(binary.BigEndian.Uint64(resp.Extras[8:]))
}
res := &StoreResult{
Cas: Cas(resp.Cas),
MutationToken: mutToken,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var duraLevelFrame *memd.DurabilityLevelFrame
var duraTimeoutFrame *memd.DurabilityTimeoutFrame
if opts.DurabilityLevel > 0 {
if crud.featureVerifier.HasBucketCapabilityStatus(BucketCapabilityDurableWrites, BucketCapabilityStatusUnsupported) {
return nil, errFeatureNotAvailable
}
duraLevelFrame = &memd.DurabilityLevelFrame{
DurabilityLevel: opts.DurabilityLevel,
}
duraTimeoutFrame = &memd.DurabilityTimeoutFrame{
DurabilityTimeout: opts.DurabilityLevelTimeout,
}
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
var preserveExpiryFrame *memd.PreserveExpiryFrame
if opts.PreserveExpiry {
preserveExpiryFrame = &memd.PreserveExpiryFrame{}
}
extraBuf := make([]byte, 8)
binary.BigEndian.PutUint32(extraBuf[0:], opts.Flags)
binary.BigEndian.PutUint32(extraBuf[4:], opts.Expiry)
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: opcode,
Datatype: opts.Datatype,
Cas: uint64(opts.Cas),
Extras: extraBuf,
Key: opts.Key,
Value: opts.Value,
DurabilityLevelFrame: duraLevelFrame,
DurabilityTimeoutFrame: duraTimeoutFrame,
UserImpersonationFrame: userFrame,
CollectionID: opts.CollectionID,
PreserveExpiryFrame: preserveExpiryFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, opName, errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) Set(opts SetOptions, cb StoreCallback) (PendingOp, error) {
return crud.store("Set", memd.CmdSet, storeOptions{
Key: opts.Key,
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
Value: opts.Value,
Flags: opts.Flags,
Datatype: opts.Datatype,
Cas: 0,
Expiry: opts.Expiry,
TraceContext: opts.TraceContext,
DurabilityLevel: opts.DurabilityLevel,
DurabilityLevelTimeout: opts.DurabilityLevelTimeout,
CollectionID: opts.CollectionID,
Deadline: opts.Deadline,
User: opts.User,
PreserveExpiry: opts.PreserveExpiry,
}, cb)
}
func (crud *crudComponent) Add(opts AddOptions, cb StoreCallback) (PendingOp, error) {
return crud.store("Add", memd.CmdAdd, storeOptions{
Key: opts.Key,
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
Value: opts.Value,
Flags: opts.Flags,
Datatype: opts.Datatype,
Cas: 0,
Expiry: opts.Expiry,
TraceContext: opts.TraceContext,
DurabilityLevel: opts.DurabilityLevel,
DurabilityLevelTimeout: opts.DurabilityLevelTimeout,
CollectionID: opts.CollectionID,
Deadline: opts.Deadline,
User: opts.User,
}, cb)
}
func (crud *crudComponent) Replace(opts ReplaceOptions, cb StoreCallback) (PendingOp, error) {
if opts.PreserveExpiry && opts.Expiry > 0 {
return nil, wrapError(errInvalidArgument, "cannot use preserve expiry and an expiry > 0 for replace")
}
return crud.store("Replace", memd.CmdReplace, storeOptions(opts), cb)
}
func (crud *crudComponent) adjoin(opName string, opcode memd.CmdCode, opts AdjoinOptions, cb AdjoinCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, opName, opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
mutToken := MutationToken{}
if len(resp.Extras) >= 16 {
mutToken.VbID = req.Vbucket
mutToken.VbUUID = VbUUID(binary.BigEndian.Uint64(resp.Extras[0:]))
mutToken.SeqNo = SeqNo(binary.BigEndian.Uint64(resp.Extras[8:]))
}
res := &AdjoinResult{
Cas: Cas(resp.Cas),
MutationToken: mutToken,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var duraLevelFrame *memd.DurabilityLevelFrame
var duraTimeoutFrame *memd.DurabilityTimeoutFrame
if opts.DurabilityLevel > 0 {
if crud.featureVerifier.HasBucketCapabilityStatus(BucketCapabilityDurableWrites, BucketCapabilityStatusUnsupported) {
return nil, errFeatureNotAvailable
}
duraLevelFrame = &memd.DurabilityLevelFrame{
DurabilityLevel: opts.DurabilityLevel,
}
duraTimeoutFrame = &memd.DurabilityTimeoutFrame{
DurabilityTimeout: opts.DurabilityLevelTimeout,
}
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
var preserveExpiryFrame *memd.PreserveExpiryFrame
if opts.PreserveExpiry {
preserveExpiryFrame = &memd.PreserveExpiryFrame{}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: opcode,
Datatype: 0,
Cas: uint64(opts.Cas),
Extras: nil,
Key: opts.Key,
Value: opts.Value,
DurabilityLevelFrame: duraLevelFrame,
DurabilityTimeoutFrame: duraTimeoutFrame,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
PreserveExpiryFrame: preserveExpiryFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, opName, errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) Append(opts AdjoinOptions, cb AdjoinCallback) (PendingOp, error) {
return crud.adjoin("Append", memd.CmdAppend, opts, cb)
}
func (crud *crudComponent) Prepend(opts AdjoinOptions, cb AdjoinCallback) (PendingOp, error) {
return crud.adjoin("Prepend", memd.CmdPrepend, opts, cb)
}
func (crud *crudComponent) counter(opName string, opcode memd.CmdCode, opts CounterOptions, cb CounterCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, opName, opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
if len(resp.Value) != 8 {
tracer.Finish()
cb(nil, errProtocol)
return
}
intVal := binary.BigEndian.Uint64(resp.Value)
mutToken := MutationToken{}
if len(resp.Extras) >= 16 {
mutToken.VbID = req.Vbucket
mutToken.VbUUID = VbUUID(binary.BigEndian.Uint64(resp.Extras[0:]))
mutToken.SeqNo = SeqNo(binary.BigEndian.Uint64(resp.Extras[8:]))
}
res := &CounterResult{
Value: intVal,
Cas: Cas(resp.Cas),
MutationToken: mutToken,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
// You cannot have an expiry when you do not want to create the document.
if opts.Initial == uint64(0xFFFFFFFFFFFFFFFF) && opts.Expiry != 0 {
return nil, errInvalidArgument
}
var duraLevelFrame *memd.DurabilityLevelFrame
var duraTimeoutFrame *memd.DurabilityTimeoutFrame
if opts.DurabilityLevel > 0 {
if crud.featureVerifier.HasBucketCapabilityStatus(BucketCapabilityDurableWrites, BucketCapabilityStatusUnsupported) {
return nil, errFeatureNotAvailable
}
duraLevelFrame = &memd.DurabilityLevelFrame{
DurabilityLevel: opts.DurabilityLevel,
}
duraTimeoutFrame = &memd.DurabilityTimeoutFrame{
DurabilityTimeout: opts.DurabilityLevelTimeout,
}
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
var preserveExpiryFrame *memd.PreserveExpiryFrame
if opts.PreserveExpiry {
preserveExpiryFrame = &memd.PreserveExpiryFrame{}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
extraBuf := make([]byte, 20)
binary.BigEndian.PutUint64(extraBuf[0:], opts.Delta)
if opts.Initial != uint64(0xFFFFFFFFFFFFFFFF) {
binary.BigEndian.PutUint64(extraBuf[8:], opts.Initial)
binary.BigEndian.PutUint32(extraBuf[16:], opts.Expiry)
} else {
binary.BigEndian.PutUint64(extraBuf[8:], 0x0000000000000000)
binary.BigEndian.PutUint32(extraBuf[16:], 0xFFFFFFFF)
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: opcode,
Datatype: 0,
Cas: uint64(opts.Cas),
Extras: extraBuf,
Key: opts.Key,
Value: nil,
DurabilityLevelFrame: duraLevelFrame,
DurabilityTimeoutFrame: duraTimeoutFrame,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
PreserveExpiryFrame: preserveExpiryFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, opName, errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) Increment(opts CounterOptions, cb CounterCallback) (PendingOp, error) {
return crud.counter("Increment", memd.CmdIncrement, opts, cb)
}
func (crud *crudComponent) Decrement(opts CounterOptions, cb CounterCallback) (PendingOp, error) {
return crud.counter("Decrement", memd.CmdDecrement, opts, cb)
}
func (crud *crudComponent) GetRandom(opts GetRandomOptions, cb GetRandomCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "GetRandom", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
if len(resp.Extras) != 4 {
tracer.Finish()
cb(nil, errProtocol)
return
}
flags := binary.BigEndian.Uint32(resp.Extras[0:])
res := &GetRandomResult{
Key: resp.Key,
Value: resp.Value,
Flags: flags,
Cas: Cas(resp.Cas),
Datatype: resp.Datatype,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGetRandom,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: nil,
Value: nil,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
RetryStrategy: opts.RetryStrategy,
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "GetRandom", errUnambiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) GetMeta(opts GetMetaOptions, cb GetMetaCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "GetMeta", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
if len(resp.Extras) != 21 {
tracer.Finish()
cb(nil, errProtocol)
return
}
res := &GetMetaResult{
Value: resp.Value,
Cas: Cas(resp.Cas),
}
res.Deleted = binary.BigEndian.Uint32(resp.Extras[0:])
res.Flags = binary.BigEndian.Uint32(resp.Extras[4:])
res.Expiry = binary.BigEndian.Uint32(resp.Extras[8:])
res.SeqNo = SeqNo(binary.BigEndian.Uint64(resp.Extras[12:]))
res.Datatype = resp.Extras[20]
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
extraBuf := make([]byte, 1)
extraBuf[0] = 2
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGetMeta,
Datatype: 0,
Cas: 0,
Extras: extraBuf,
Key: opts.Key,
Value: nil,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "GetMeta", errUnambiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) SetMeta(opts SetMetaOptions, cb SetMetaCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "SetMeta", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
mutToken := MutationToken{}
if len(resp.Extras) >= 16 {
mutToken.VbID = req.Vbucket
mutToken.VbUUID = VbUUID(binary.BigEndian.Uint64(resp.Extras[0:]))
mutToken.SeqNo = SeqNo(binary.BigEndian.Uint64(resp.Extras[8:]))
}
res := &SetMetaResult{
Cas: Cas(resp.Cas),
MutationToken: mutToken,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
extraBuf := make([]byte, 30+len(opts.Extra))
binary.BigEndian.PutUint32(extraBuf[0:], opts.Flags)
binary.BigEndian.PutUint32(extraBuf[4:], opts.Expiry)
binary.BigEndian.PutUint64(extraBuf[8:], opts.RevNo)
binary.BigEndian.PutUint64(extraBuf[16:], uint64(opts.Cas))
binary.BigEndian.PutUint32(extraBuf[24:], opts.Options)
binary.BigEndian.PutUint16(extraBuf[28:], uint16(len(opts.Extra)))
copy(extraBuf[30:], opts.Extra)
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdSetMeta,
Datatype: opts.Datatype,
Cas: 0,
Extras: extraBuf,
Key: opts.Key,
Value: opts.Value,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "SetMeta", errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) DeleteMeta(opts DeleteMetaOptions, cb DeleteMetaCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "DeleteMeta", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
mutToken := MutationToken{}
if len(resp.Extras) >= 16 {
mutToken.VbID = req.Vbucket
mutToken.VbUUID = VbUUID(binary.BigEndian.Uint64(resp.Extras[0:]))
mutToken.SeqNo = SeqNo(binary.BigEndian.Uint64(resp.Extras[8:]))
}
res := &DeleteMetaResult{
Cas: Cas(resp.Cas),
MutationToken: mutToken,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
extraBuf := make([]byte, 30+len(opts.Extra))
binary.BigEndian.PutUint32(extraBuf[0:], opts.Flags)
binary.BigEndian.PutUint32(extraBuf[4:], opts.Expiry)
binary.BigEndian.PutUint64(extraBuf[8:], opts.RevNo)
binary.BigEndian.PutUint64(extraBuf[16:], uint64(opts.Cas))
binary.BigEndian.PutUint32(extraBuf[24:], opts.Options)
binary.BigEndian.PutUint16(extraBuf[28:], uint16(len(opts.Extra)))
copy(extraBuf[30:], opts.Extra)
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdDelMeta,
Datatype: opts.Datatype,
Cas: 0,
Extras: extraBuf,
Key: opts.Key,
Value: opts.Value,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "DeleteMeta", errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
gocbcore-10.2.3/crudcomponent_rangescan.go 0000664 0000000 0000000 00000025572 14417540156 0020632 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/binary"
"encoding/json"
"fmt"
"github.com/golang/snappy"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
type rangeScanCreateRequest struct {
Collection string `json:"collection,omitempty"`
KeyOnly bool `json:"key_only,omitempty"`
Range *rangeScanCreateRange `json:"range,omitempty"`
Sampling *rangeScanCreateSample `json:"sampling,omitempty"`
Snapshot *rangeScanCreateSnapshot `json:"snapshot_requirements,omitempty"`
}
type rangeScanCreateRange struct {
Start string `json:"start,omitempty"`
End string `json:"end,omitempty"`
ExclusiveStart string `json:"excl_start,omitempty"`
ExclusiveEnd string `json:"excl_end,omitempty"`
}
type rangeScanCreateSample struct {
Seed uint64 `json:"seed,omitempty"`
Samples uint64 `json:"samples"`
}
type rangeScanCreateSnapshot struct {
VbUUID string `json:"vb_uuid"`
SeqNo uint64 `json:"seqno"`
SeqNoExists bool `json:"seqno_exists,omitempty"`
Timeout uint64 `json:"timeout_ms,omitempty"`
}
func (crud *crudComponent) RangeScanCreate(vbID uint16, opts RangeScanCreateOptions, cb RangeScanCreateCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "RangeScanCreate", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
res := RangeScanCreateResult{}
res.ScanUUUID = resp.Value
res.KeysOnly = opts.KeysOnly
tracer.Finish()
cb(&res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
createReq, err := opts.toRequest()
if err != nil {
return nil, err
}
value, err := json.Marshal(createReq)
if err != nil {
return nil, err
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdRangeScanCreate,
Datatype: uint8(memd.DatatypeFlagJSON),
Cas: 0,
Extras: nil,
Key: nil,
Value: value,
UserImpersonationFrame: userFrame,
Vbucket: vbID,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
RetryStrategy: opts.RetryStrategy,
ScopeName: opts.ScopeName,
CollectionName: opts.CollectionName,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallbackAndFinishTracer(&TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "RangeScanCreate",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
}, tracer)
}))
}
return op, nil
}
func (crud *crudComponent) RangeScanContinue(scanUUID []byte, vbID uint16, opts RangeScanContinueOptions, dataCb RangeScanContinueDataCallback,
actionCb RangeScanContinueActionCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "RangeScanContinue", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
actionCb(nil, err)
return
}
if len(resp.Extras) != 4 {
tracer.Finish()
actionCb(nil, errProtocol)
return
}
keysOnlyFlag := binary.BigEndian.Uint32(resp.Extras[0:])
items, err := parseRangeScanData(resp.Value, keysOnlyFlag == 0, crud.disableDecompression)
if err != nil {
tracer.Finish()
actionCb(nil, err)
return
}
if len(resp.Value) > 0 {
dataCb(items)
}
res := RangeScanContinueResult{
More: resp.Status == memd.StatusRangeScanMore,
Complete: resp.Status == memd.StatusRangeScanComplete,
}
if res.More || res.Complete {
// This is effectively the same as calling cancelReqTrace, this will set the cmd and net spans to
// nil on the request - meaning that the internal cancel below will not cause issues when it calls
// cancelReqTrace.
stopNetTrace(req, resp, resp.remoteAddr, resp.sourceAddr)
stopCmdTrace(req)
// As this is a persistent request, we must manually cancel it to remove
// it from the pending ops list.
req.internalCancel(nil)
tracer.Finish()
actionCb(&res, nil)
}
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
if len(scanUUID) != 16 {
return nil, wrapError(errInvalidArgument, fmt.Sprintf("scanUUID must be 16 bytes, was %d", len(scanUUID)))
}
var deadlineMs uint32
if !opts.Deadline.IsZero() {
deadlineMs = uint32(time.Until(opts.Deadline).Milliseconds())
}
extraBuf := make([]byte, 28)
copy(extraBuf[:16], scanUUID)
binary.BigEndian.PutUint32(extraBuf[16:], opts.MaxCount)
binary.BigEndian.PutUint32(extraBuf[20:], deadlineMs)
binary.BigEndian.PutUint32(extraBuf[24:], opts.MaxBytes)
// Note that collection and scope aren't used here. That means that on a collection unknown from the server
// we will not attempt to refresh the CID.
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdRangeScanContinue,
Cas: 0,
Extras: extraBuf,
Key: nil,
Value: nil,
UserImpersonationFrame: userFrame,
Vbucket: vbID,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
RetryStrategy: opts.RetryStrategy,
Persistent: true,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallbackAndFinishTracer(&TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "RangeScanContinue",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
}, tracer)
}))
}
return op, nil
}
func (crud *crudComponent) RangeScanCancel(scanUUID []byte, vbID uint16, opts RangeScanCancelOptions, cb RangeScanCancelCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "RangeScanCancel", opts.TraceContext)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
tracer.Finish()
cb(&RangeScanCancelResult{}, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
if len(scanUUID) != 16 {
return nil, wrapError(errInvalidArgument, fmt.Sprintf("scanUUID must be 16 bytes, was %d", len(scanUUID)))
}
extraBuf := make([]byte, 16)
copy(extraBuf[:16], scanUUID)
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdRangeScanCancel,
Cas: 0,
Extras: extraBuf,
Key: nil,
Value: nil,
UserImpersonationFrame: userFrame,
Vbucket: vbID,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallbackAndFinishTracer(&TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "RangeScanCreate",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
}, tracer)
}))
}
return op, nil
}
func parseRangeScanData(data []byte, keysOnly bool, disableDecompression bool) ([]RangeScanItem, error) {
if keysOnly {
return parseRangeScanKeys(data), nil
}
return parseRangeScanDocs(data, disableDecompression)
}
func parseRangeScanLebEncoded(data []byte) ([]byte, uint64) {
keyLen, n := binary.Uvarint(data)
keyLen = keyLen + uint64(n)
return data[uint64(n):keyLen], keyLen
}
func parseRangeScanKeys(data []byte) []RangeScanItem {
var keys []RangeScanItem
var i uint64
dataLen := uint64(len(data))
for {
if i >= dataLen {
break
}
key, n := parseRangeScanLebEncoded(data[i:])
keys = append(keys, RangeScanItem{
Key: key,
})
i = i + n
}
return keys
}
func parseRangeScanItem(data []byte, disableDecompression bool) (RangeScanItem, uint64, error) {
flags := binary.BigEndian.Uint32(data[0:])
expiry := binary.BigEndian.Uint32(data[4:])
seqno := binary.BigEndian.Uint64(data[8:])
cas := binary.BigEndian.Uint64(data[16:])
datatype := data[24]
key, n := parseRangeScanLebEncoded(data[25:])
value, n2 := parseRangeScanLebEncoded(data[25+n:])
isCompressed := (datatype & uint8(memd.DatatypeFlagCompressed)) != 0
if isCompressed && !disableDecompression {
newValue, err := snappy.Decode(nil, value)
if err != nil {
return RangeScanItem{}, 0, nil
}
value = newValue
datatype = datatype & ^uint8(memd.DatatypeFlagCompressed)
}
return RangeScanItem{
Value: value,
Key: key,
Flags: flags,
Cas: Cas(cas),
Expiry: expiry,
SeqNo: SeqNo(seqno),
Datatype: datatype,
}, 25 + n + n2, nil
}
func parseRangeScanDocs(data []byte, disableDecompression bool) ([]RangeScanItem, error) {
var items []RangeScanItem
var i uint64
dataLen := uint64(len(data))
for {
if i >= dataLen {
break
}
item, n, err := parseRangeScanItem(data[i:], disableDecompression)
if err != nil {
return nil, err
}
items = append(items, item)
i = i + n
}
return items, nil
}
gocbcore-10.2.3/crudcomponent_rangescan_test.go 0000664 0000000 0000000 00000033054 14417540156 0021663 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"github.com/couchbase/gocbcore/v10/memd"
"github.com/google/uuid"
"strings"
"time"
)
type rangeScanMutation struct {
cas Cas
mutationToken MutationToken
}
type rangeScanMutations struct {
muts map[string]rangeScanMutation
vbuuid VbUUID
highSeqNo SeqNo
}
func (suite *StandardTestSuite) setupRangeScan(docIDs []string, value []byte, collection, scope string) *rangeScanMutations {
agent, s := suite.GetAgentAndHarness()
muts := &rangeScanMutations{
muts: make(map[string]rangeScanMutation),
}
for i := 0; i < len(docIDs); i++ {
s.PushOp(agent.Set(SetOptions{
Key: []byte(docIDs[i]),
Value: value,
ScopeName: scope,
CollectionName: collection,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
muts.muts[docIDs[i]] = rangeScanMutation{
cas: res.Cas,
mutationToken: res.MutationToken,
}
if res.MutationToken.SeqNo > muts.highSeqNo {
muts.highSeqNo = res.MutationToken.SeqNo
muts.vbuuid = res.MutationToken.VbUUID
}
})
}))
s.Wait(0)
}
suite.tracer.Reset()
suite.meter.Reset()
return muts
}
func (suite *StandardTestSuite) TestRangeScanRangeLargeValues() {
suite.EnsureSupportsFeature(TestFeatureRangeScan)
agent, _ := suite.GetAgentAndHarness()
size := 8192 * 2
value := make([]byte, size)
for i := 0; i < size; i++ {
value[i] = byte(i)
}
docIDs := []string{"largevalues-2960", "largevalues-3064", "largevalues-3686", "largevalues-3716", "largevalues-5354",
"largevalues-5426", "largevalues-6175", "largevalues-6607", "largevalues-6797", "largevalues-7871"}
muts := suite.setupRangeScan(docIDs, value, "", "")
data := suite.doRangeScan(12,
RangeScanCreateOptions{
Range: &RangeScanCreateRangeScanConfig{
Start: []byte("largevalues"),
End: []byte("largevalues\xFF"),
},
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
},
RangeScanContinueOptions{
Deadline: time.Now().Add(10 * time.Second),
},
)
itemsMap := make(map[string]RangeScanItem)
for _, item := range data {
itemsMap[string(item.Key)] = item
}
for id, mut := range muts.muts {
item, ok := itemsMap[id]
if suite.Assert().True(ok) {
suite.Assert().Equal(mut.cas, item.Cas)
suite.Assert().Equal(mut.mutationToken.SeqNo, item.SeqNo)
suite.Assert().Equal(value, item.Value)
}
}
suite.verifyRangeScanTelemetry(agent)
}
func (suite *StandardTestSuite) TestRangeScanRangeSmallValues() {
suite.EnsureSupportsFeature(TestFeatureRangeScan)
agent, _ := suite.GetAgentAndHarness()
value := []byte(`{"barry": "sheen"}`)
docIDs := []string{"rangesmallvalues-1023", "rangesmallvalues-1751", "rangesmallvalues-2202",
"rangesmallvalues-2392", "rangesmallvalues-2570", "rangesmallvalues-4132", "rangesmallvalues-4640",
"rangesmallvalues-5836", "rangesmallvalues-7283", "rangesmallvalues-7313"}
muts := suite.setupRangeScan(docIDs, value, "", "")
data := suite.doRangeScan(12,
RangeScanCreateOptions{
Range: &RangeScanCreateRangeScanConfig{
Start: []byte("rangesmallvalues"),
End: []byte("rangesmallvalues\xFF"),
},
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
},
RangeScanContinueOptions{
Deadline: time.Now().Add(10 * time.Second),
},
)
itemsMap := make(map[string]RangeScanItem)
for _, item := range data {
itemsMap[string(item.Key)] = item
}
for id, mut := range muts.muts {
item, ok := itemsMap[id]
if suite.Assert().True(ok) {
suite.Assert().Equal(mut.cas, item.Cas)
suite.Assert().Equal(mut.mutationToken.SeqNo, item.SeqNo)
suite.Assert().Equal(value, item.Value)
}
}
suite.verifyRangeScanTelemetry(agent)
}
func (suite *StandardTestSuite) TestRangeScanRangeCollectionRetry() {
suite.EnsureSupportsFeature(TestFeatureRangeScan)
agent, _ := suite.GetAgentAndHarness()
collectionName := strings.Replace(uuid.NewString(), "-", "", -1)
_, err := testCreateCollection(collectionName, suite.ScopeName, suite.BucketName, agent)
suite.Require().Nil(err, err)
defer testDeleteCollection(collectionName, suite.ScopeName, suite.BucketName, agent, false)
value := "value"
docIDs := []string{"rangecollectionretry-9695", "rangecollectionretry-24520", "rangecollectionretry-90825",
"rangecollectionretry-119677", "rangecollectionretry-150939", "rangecollectionretry-170176",
"rangecollectionretry-199557", "rangecollectionretry-225568", "rangecollectionretry-231302",
"rangecollectionretry-245898"}
muts := suite.setupRangeScan(docIDs, []byte(value), collectionName, suite.ScopeName)
// We're going to force a refresh so we need to delete the collection from our cache.
agent.collections.mapLock.Lock()
delete(agent.collections.idMap, suite.ScopeName+"."+collectionName)
agent.collections.mapLock.Unlock()
data := suite.doRangeScan(12,
RangeScanCreateOptions{
Range: &RangeScanCreateRangeScanConfig{
Start: []byte("rangecollectionretry"),
End: []byte("rangecollectionretry\xFF"),
},
KeysOnly: true,
ScopeName: suite.ScopeName,
CollectionName: collectionName,
Snapshot: &RangeScanCreateSnapshotRequirements{
VbUUID: muts.vbuuid,
SeqNo: muts.highSeqNo,
},
},
RangeScanContinueOptions{
Deadline: time.Now().Add(10 * time.Second),
},
)
itemsMap := make(map[string]RangeScanItem)
for _, item := range data {
itemsMap[string(item.Key)] = item
}
for id := range muts.muts {
item, ok := itemsMap[id]
if suite.Assert().True(ok) {
suite.Assert().Zero(item.Cas)
suite.Assert().Zero(item.SeqNo)
suite.Assert().Empty(item.Value)
}
}
suite.verifyRangeScanTelemetry(agent)
}
func (suite *StandardTestSuite) TestRangeScanRangeKeysOnly() {
suite.EnsureSupportsFeature(TestFeatureRangeScan)
agent, _ := suite.GetAgentAndHarness()
value := "value"
docIDs := []string{"rangekeysonly-1269", "rangekeysonly-2048", "rangekeysonly-4378", "rangekeysonly-7159",
"rangekeysonly-8898", "rangekeysonly-8908", "rangekeysonly-19559", "rangekeysonly-20808",
"rangekeysonly-20998", "rangekeysonly-25889"}
muts := suite.setupRangeScan(docIDs, []byte(value), "", "")
data := suite.doRangeScan(12,
RangeScanCreateOptions{
Range: &RangeScanCreateRangeScanConfig{
Start: []byte("rangekeysonly"),
End: []byte("rangekeysonly\xFF"),
},
KeysOnly: true,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
},
RangeScanContinueOptions{
Deadline: time.Now().Add(10 * time.Second),
},
)
itemsMap := make(map[string]RangeScanItem)
for _, item := range data {
itemsMap[string(item.Key)] = item
}
for id := range muts.muts {
item, ok := itemsMap[id]
if suite.Assert().True(ok) {
suite.Assert().Zero(item.Cas)
suite.Assert().Zero(item.SeqNo)
suite.Assert().Empty(item.Value)
}
}
suite.verifyRangeScanTelemetry(agent)
}
func (suite *StandardTestSuite) TestRangeScanSamplingKeysOnly() {
suite.EnsureSupportsFeature(TestFeatureRangeScan)
agent, _ := suite.GetAgentAndHarness()
scopeName := "rangeScanSampleKeysOnly"
collectionName := "rangeScan"
_, err := testCreateScope(scopeName, suite.BucketName, agent)
suite.Require().Nil(err, err)
defer testDeleteScope(scopeName, suite.BucketName, agent, false)
_, err = testCreateCollection(collectionName, scopeName, suite.BucketName, agent)
suite.Require().Nil(err, err)
defer testDeleteCollection(collectionName, scopeName, suite.BucketName, agent, false)
value := "value"
docIDs := []string{"samplescankeys-170", "samplescankeys-602", "samplescankeys-792", "samplescankeys-3978",
"samplescankeys-6869", "samplescankeys-9038", "samplescankeys-10806", "samplescankeys-10996",
"samplescankeys-11092", "samplescankeys-11102"}
muts := suite.setupRangeScan(docIDs, []byte(value), collectionName, scopeName)
data := suite.doRangeScan(12,
RangeScanCreateOptions{
Sampling: &RangeScanCreateRandomSamplingConfig{
Samples: 10,
},
KeysOnly: true,
ScopeName: scopeName,
CollectionName: collectionName,
},
RangeScanContinueOptions{
Deadline: time.Now().Add(10 * time.Second),
},
)
itemsMap := make(map[string]RangeScanItem)
for _, item := range data {
itemsMap[string(item.Key)] = item
}
for id := range muts.muts {
item, ok := itemsMap[id]
if suite.Assert().True(ok) {
suite.Assert().Zero(item.Cas)
suite.Assert().Zero(item.SeqNo)
suite.Assert().Empty(item.Value)
}
}
suite.verifyRangeScanTelemetry(agent)
}
func (suite *StandardTestSuite) TestRangeScanRangeSnapshot() {
suite.EnsureSupportsFeature(TestFeatureRangeScan)
agent, s := suite.GetAgentAndHarness()
value := "value"
docIDs := []string{"rangescansnapshot-38523", "rangescansnapshot-45448", "rangescansnapshot-51222",
"rangescansnapshot-108547", "rangescansnapshot-135193", "rangescansnapshot-161246", "rangescansnapshot-188667",
"rangescansnapshot-220032", "rangescansnapshot-234658", "rangescansnapshot-249733"}
var highSeqNo SeqNo
var vbUUID VbUUID
for i := 0; i < len(docIDs); i++ {
s.PushOp(agent.Set(SetOptions{
Key: []byte(docIDs[i]),
Value: []byte(value),
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
highSeqNo = res.MutationToken.SeqNo
vbUUID = res.MutationToken.VbUUID
})
}))
s.Wait(0)
}
suite.tracer.Reset()
suite.meter.Reset()
data := suite.doRangeScan(12,
RangeScanCreateOptions{
Range: &RangeScanCreateRangeScanConfig{
Start: []byte("rangescansnapshot"),
End: []byte("rangescansnapshot\xFF"),
},
Snapshot: &RangeScanCreateSnapshotRequirements{
VbUUID: vbUUID,
SeqNo: highSeqNo,
},
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
},
RangeScanContinueOptions{
Deadline: time.Now().Add(10 * time.Second),
},
)
itemsMap := make(map[string]RangeScanItem)
for _, item := range data {
itemsMap[string(item.Key)] = item
}
for _, id := range docIDs {
item, ok := itemsMap[id]
if suite.Assert().True(ok) {
suite.Assert().NotZero(item.Cas)
suite.Assert().NotZero(item.SeqNo)
suite.Assert().Equal([]byte(value), item.Value)
}
}
suite.verifyRangeScanTelemetry(agent)
}
func (suite *StandardTestSuite) TestRangeScanRangeCancellation() {
suite.EnsureSupportsFeature(TestFeatureRangeScan)
agent, s := suite.GetAgentAndHarness()
value := "value"
docIDs := []string{"rangescancancel-2746", "rangescancancel-37795", "rangescancancel-63440", "rangescancancel-116036",
"rangescancancel-136879", "rangescancancel-156589", "rangescancancel-196316", "rangescancancel-203197",
"rangescancancel-243428", "rangescancancel-257242"}
var highSeqNo SeqNo
var vbUUID VbUUID
for i := 0; i < len(docIDs); i++ {
s.PushOp(agent.Set(SetOptions{
Key: []byte(docIDs[i]),
Value: []byte(value),
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
highSeqNo = res.MutationToken.SeqNo
vbUUID = res.MutationToken.VbUUID
})
}))
s.Wait(0)
}
suite.tracer.Reset()
suite.meter.Reset()
var scanUUID []byte
s.PushOp(agent.RangeScanCreate(12, RangeScanCreateOptions{
Range: &RangeScanCreateRangeScanConfig{
Start: []byte("rangescancancel"),
End: []byte("rangescancancel\xFF"),
},
Snapshot: &RangeScanCreateSnapshotRequirements{
VbUUID: vbUUID,
SeqNo: highSeqNo,
},
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(res *RangeScanCreateResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("RangeScanCreate operation failed: %v", err)
}
scanUUID = res.ScanUUUID
})
}))
s.Wait(0)
s.PushOp(agent.RangeScanCancel(scanUUID, 12, RangeScanCancelOptions{}, func(result *RangeScanCancelResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("RangeScanCancel operation failed: %v", err)
}
})
}))
s.Wait(0)
}
func (suite *StandardTestSuite) verifyRangeScanTelemetry(agent *Agent) {
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(2, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "RangeScanCreate", agent.BucketName(), memd.CmdRangeScanCreate.Name(), 1, false, "")
suite.AssertOpSpan(nilParents[1], "RangeScanContinue", agent.BucketName(), memd.CmdRangeScanContinue.Name(), 1, false, "")
}
}
suite.VerifyKVMetrics(suite.meter, "RangeScanCreate", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "RangeScanContinue", 1, false, false)
}
func (suite *StandardTestSuite) doRangeScan(vbID uint16, opts RangeScanCreateOptions,
contOpts RangeScanContinueOptions) []RangeScanItem {
agent, s := suite.GetAgentAndHarness()
var scanUUID []byte
s.PushOp(agent.RangeScanCreate(vbID, opts, func(res *RangeScanCreateResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("RangeScanCreate operation failed: %v", err)
}
scanUUID = res.ScanUUUID
})
}))
s.Wait(0)
var data []RangeScanItem
for {
more := make(chan struct{}, 1)
s.PushOp(agent.RangeScanContinue(scanUUID, vbID, contOpts, func(items []RangeScanItem) {
data = append(data, items...)
}, func(res *RangeScanContinueResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("RangeScanContinue operation failed: %v", err)
return
}
if res.Complete {
close(more)
return
}
if res.More {
more <- struct{}{}
}
})
}))
s.Wait(0)
_, cont := <-more
if !cont {
break
}
}
return data
}
gocbcore-10.2.3/crudcomponent_subdoc.go 0000664 0000000 0000000 00000026627 14417540156 0020152 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/binary"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
type subdocOpList struct {
ops []SubDocOp
indexes []int
}
func (sol *subdocOpList) Reorder(ops []SubDocOp) {
var xAttrOps []SubDocOp
var xAttrIndexes []int
var sops []SubDocOp
var opIndexes []int
for i, op := range ops {
if op.Flags&memd.SubdocFlagXattrPath != 0 {
xAttrOps = append(xAttrOps, op)
xAttrIndexes = append(xAttrIndexes, i)
} else {
sops = append(sops, op)
opIndexes = append(opIndexes, i)
}
}
sol.ops = append(xAttrOps, sops...)
sol.indexes = append(xAttrIndexes, opIndexes...)
}
func (crud *crudComponent) LookupIn(opts LookupInOptions, cb LookupInCallback) (PendingOp, error) {
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "LookupIn", opts.TraceContext)
results := make([]SubDocResult, len(opts.Ops))
var subdocs subdocOpList
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil &&
!isErrorStatus(err, memd.StatusSubDocMultiPathFailureDeleted) &&
!isErrorStatus(err, memd.StatusSubDocSuccessDeleted) &&
!isErrorStatus(err, memd.StatusSubDocBadMulti) {
tracer.Finish()
cb(nil, err)
return
}
respIter := 0
for i := range results {
if respIter+6 > len(resp.Value) {
tracer.Finish()
cb(nil, errProtocol)
return
}
resError := memd.StatusCode(binary.BigEndian.Uint16(resp.Value[respIter+0:]))
resValueLen := int(binary.BigEndian.Uint32(resp.Value[respIter+2:]))
if respIter+6+resValueLen > len(resp.Value) {
tracer.Finish()
cb(nil, errProtocol)
return
}
if resError != memd.StatusSuccess {
results[subdocs.indexes[i]].Err = crud.makeSubDocError(i, resError, req, resp)
}
results[subdocs.indexes[i]].Value = resp.Value[respIter+6 : respIter+6+resValueLen]
respIter += 6 + resValueLen
}
res := &LookupInResult{
Cas: Cas(resp.Cas),
Ops: results,
}
res.Internal.IsDeleted = isErrorStatus(err, memd.StatusSubDocSuccessDeleted) ||
isErrorStatus(err, memd.StatusSubDocMultiPathFailureDeleted)
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
subdocs.Reorder(opts.Ops)
pathBytesList := make([][]byte, len(opts.Ops))
pathBytesTotal := 0
for i, op := range subdocs.ops {
pathBytes := []byte(op.Path)
pathBytesList[i] = pathBytes
pathBytesTotal += len(pathBytes)
}
valueBuf := make([]byte, len(opts.Ops)*4+pathBytesTotal)
valueIter := 0
for i, op := range subdocs.ops {
if op.Op != memd.SubDocOpGet && op.Op != memd.SubDocOpExists &&
op.Op != memd.SubDocOpGetDoc && op.Op != memd.SubDocOpGetCount {
return nil, errInvalidArgument
}
if op.Value != nil {
return nil, errInvalidArgument
}
pathBytes := pathBytesList[i]
pathBytesLen := len(pathBytes)
valueBuf[valueIter+0] = uint8(op.Op)
valueBuf[valueIter+1] = uint8(op.Flags)
binary.BigEndian.PutUint16(valueBuf[valueIter+2:], uint16(pathBytesLen))
copy(valueBuf[valueIter+4:], pathBytes)
valueIter += 4 + pathBytesLen
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
var extraBuf []byte
if opts.Flags != 0 {
extraBuf = append(extraBuf, uint8(opts.Flags))
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdSubDocMultiLookup,
Datatype: 0,
Cas: 0,
Extras: extraBuf,
Key: opts.Key,
Value: valueBuf,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "LookupIn", errUnambiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) MutateIn(opts MutateInOptions, cb MutateInCallback) (PendingOp, error) {
if len(opts.Ops) == 0 {
return nil, wrapError(errInvalidArgument, "at least one op must be present")
}
tracer := crud.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "MutateIn", opts.TraceContext)
results := make([]SubDocResult, len(opts.Ops))
var subdocs subdocOpList
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
// GOCBC-1356: memcached can return a NOT_STORED response when inserting a doc with sub-doc.
if isErrorStatus(err, memd.StatusNotStored) && opts.Flags&memd.SubdocDocFlagAddDoc != 0 {
tracer.Finish()
cb(nil, crud.errMapManager.EnhanceKvError(errDocumentExists, resp, req))
return
}
if err != nil &&
!isErrorStatus(err, memd.StatusSubDocSuccessDeleted) &&
!isErrorStatus(err, memd.StatusSubDocBadMulti) {
tracer.Finish()
cb(nil, err)
return
}
if isErrorStatus(err, memd.StatusSubDocBadMulti) {
if len(resp.Value) != 3 {
tracer.Finish()
cb(nil, errProtocol)
return
}
opIndex := int(resp.Value[0])
resError := memd.StatusCode(binary.BigEndian.Uint16(resp.Value[1:]))
err := crud.makeSubDocError(opIndex, resError, req, resp)
tracer.Finish()
cb(nil, err)
return
}
for readPos := uint32(0); readPos < uint32(len(resp.Value)); {
opIndex := int(resp.Value[readPos+0])
opStatus := memd.StatusCode(binary.BigEndian.Uint16(resp.Value[readPos+1:]))
results[subdocs.indexes[opIndex]].Err = crud.makeSubDocError(opIndex, opStatus, req, resp)
readPos += 3
if opStatus == memd.StatusSuccess {
valLength := binary.BigEndian.Uint32(resp.Value[readPos:])
results[subdocs.indexes[opIndex]].Value = resp.Value[readPos+4 : readPos+4+valLength]
readPos += 4 + valLength
}
}
mutToken := MutationToken{}
if len(resp.Extras) >= 16 {
mutToken.VbID = req.Vbucket
mutToken.VbUUID = VbUUID(binary.BigEndian.Uint64(resp.Extras[0:]))
mutToken.SeqNo = SeqNo(binary.BigEndian.Uint64(resp.Extras[8:]))
}
res := &MutateInResult{
Cas: Cas(resp.Cas),
MutationToken: mutToken,
Ops: results,
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var duraLevelFrame *memd.DurabilityLevelFrame
var duraTimeoutFrame *memd.DurabilityTimeoutFrame
if opts.DurabilityLevel > 0 {
if crud.featureVerifier.HasBucketCapabilityStatus(BucketCapabilityDurableWrites, BucketCapabilityStatusUnsupported) {
return nil, errFeatureNotAvailable
}
duraLevelFrame = &memd.DurabilityLevelFrame{
DurabilityLevel: opts.DurabilityLevel,
}
duraTimeoutFrame = &memd.DurabilityTimeoutFrame{
DurabilityTimeout: opts.DurabilityLevelTimeout,
}
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
var preserveExpiryFrame *memd.PreserveExpiryFrame
if opts.PreserveExpiry {
if opts.Flags|memd.SubdocDocFlagAddDoc == 1 {
return nil, wrapError(errInvalidArgument, "cannot use preserve expiry with add doc flags")
}
if opts.Expiry != 0 && opts.PreserveExpiry && opts.Flags|memd.SubdocDocFlagNone == 1 {
return nil, wrapError(errInvalidArgument, "cannot use preserve expiry with expiry and no doc flags")
}
preserveExpiryFrame = &memd.PreserveExpiryFrame{}
}
if opts.Flags&memd.SubdocDocFlagCreateAsDeleted != 0 {
// We can get here before support status is actually known, we'll send the request unless we know for a fact
// that this is unsupported.
if crud.featureVerifier.HasBucketCapabilityStatus(BucketCapabilityCreateAsDeleted, BucketCapabilityStatusUnsupported) {
return nil, errFeatureNotAvailable
}
}
subdocs.Reorder(opts.Ops)
pathBytesList := make([][]byte, len(opts.Ops))
pathBytesTotal := 0
valueBytesTotal := 0
for i, op := range subdocs.ops {
pathBytes := []byte(op.Path)
pathBytesList[i] = pathBytes
pathBytesTotal += len(pathBytes)
valueBytesTotal += len(op.Value)
}
valueBuf := make([]byte, len(opts.Ops)*8+pathBytesTotal+valueBytesTotal)
valueIter := 0
for i, op := range subdocs.ops {
if op.Op != memd.SubDocOpDictAdd && op.Op != memd.SubDocOpDictSet &&
op.Op != memd.SubDocOpDelete && op.Op != memd.SubDocOpReplace &&
op.Op != memd.SubDocOpArrayPushLast && op.Op != memd.SubDocOpArrayPushFirst &&
op.Op != memd.SubDocOpArrayInsert && op.Op != memd.SubDocOpArrayAddUnique &&
op.Op != memd.SubDocOpCounter && op.Op != memd.SubDocOpSetDoc &&
op.Op != memd.SubDocOpAddDoc && op.Op != memd.SubDocOpDeleteDoc &&
op.Op != memd.SubDocOpReplaceBodyWithXattr {
return nil, errInvalidArgument
}
if op.Op == memd.SubDocOpReplaceBodyWithXattr {
// We can get here before support status is actually known, we'll send the request unless we know for a fact
// that this is unsupported.
if crud.featureVerifier.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusUnsupported) {
return nil, errFeatureNotAvailable
}
}
pathBytes := pathBytesList[i]
pathBytesLen := len(pathBytes)
valueBytesLen := len(op.Value)
valueBuf[valueIter+0] = uint8(op.Op)
valueBuf[valueIter+1] = uint8(op.Flags)
binary.BigEndian.PutUint16(valueBuf[valueIter+2:], uint16(pathBytesLen))
binary.BigEndian.PutUint32(valueBuf[valueIter+4:], uint32(valueBytesLen))
copy(valueBuf[valueIter+8:], pathBytes)
copy(valueBuf[valueIter+8+pathBytesLen:], op.Value)
valueIter += 8 + pathBytesLen + valueBytesLen
}
var extraBuf []byte
if opts.Expiry != 0 {
tmpBuf := make([]byte, 4)
binary.BigEndian.PutUint32(tmpBuf[0:], opts.Expiry)
extraBuf = append(extraBuf, tmpBuf...)
}
if opts.Flags != 0 {
extraBuf = append(extraBuf, uint8(opts.Flags))
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = crud.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdSubDocMultiMutation,
Datatype: 0,
Cas: uint64(opts.Cas),
Extras: extraBuf,
Key: opts.Key,
Value: valueBuf,
DurabilityLevelFrame: duraLevelFrame,
DurabilityTimeoutFrame: duraTimeoutFrame,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
PreserveExpiryFrame: preserveExpiryFrame,
},
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := crud.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
req.cancelWithCallbackAndFinishTracer(
makeTimeoutError(start, "MutateIn", errAmbiguousTimeout, req),
tracer,
)
}))
}
return op, nil
}
func (crud *crudComponent) makeSubDocError(index int, code memd.StatusCode, req *memdQRequest, resp *memdQResponse) error {
err := getKvStatusCodeError(code)
err = translateMemdError(err, req)
err = crud.errMapManager.EnhanceKvError(err, resp, req)
return SubDocumentError{
Index: index,
InnerError: err,
}
}
gocbcore-10.2.3/crudcomponent_subdoc_test.go 0000664 0000000 0000000 00000116265 14417540156 0021207 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"bytes"
"encoding/json"
"errors"
"github.com/couchbase/gocbcore/v10/memd"
"strconv"
"strings"
"time"
)
func (suite *StandardTestSuite) TestSubdocXattrs() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testXattr"),
Value: []byte("{\"x\":\"xattrs\"}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
})
}))
s.Wait(0)
mutateOps := []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagMkDirP,
Path: "xatest.test",
Value: []byte("\"test value\""),
},
/*{
Op: SubDocOpDictSet,
Flags: SubdocFlagXattrPath | SubdocFlagExpandMacros | SubdocFlagMkDirP,
Path: "xatest.rev",
Value: []byte("\"${Mutation.CAS}\""),
},*/
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}
s.PushOp(agent.MutateIn(MutateInOptions{
Key: []byte("testXattr"),
Ops: mutateOps,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *MutateInResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Mutate operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
lookupOps := []SubDocOp{
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagXattrPath,
Path: "xatest",
},
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagNone,
Path: "x",
},
}
s.PushOp(agent.LookupIn(LookupInOptions{
Key: []byte("testXattr"),
Ops: lookupOps,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *LookupInResult, err error) {
s.Wrap(func() {
if len(res.Ops) != 2 {
s.Fatalf("Lookup operation wrong count")
}
if res.Ops[0].Err != nil {
s.Fatalf("Lookup operation 1 failed: %v", res.Ops[0].Err)
}
if res.Ops[1].Err != nil {
s.Fatalf("Lookup operation 2 failed: %v", res.Ops[1].Err)
}
/*
xatest := fmt.Sprintf(`{"test":"test value","rev":"0x%016x"}`, cas)
if !bytes.Equal(res[0].Value, []byte(xatest)) {
s.Fatalf("Unexpected xatest value %s (doc) != %s (header)", res[0].Value, xatest)
}
*/
if !bytes.Equal(res.Ops[0].Value, []byte(`{"test":"test value"}`)) {
s.Fatalf("Unexpected xatest value %s", res.Ops[0].Value)
}
if !bytes.Equal(res.Ops[1].Value, []byte(`"x value"`)) {
s.Fatalf("Unexpected document value %s", res.Ops[1].Value)
}
})
}))
s.Wait(0)
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "MutateIn", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "LookupIn", 1, false, false)
}
func (suite *StandardTestSuite) TestSubdocXattrsReorder() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testXattrReorder"),
Value: []byte("{\"x\":\"xattrs\", \"y\":\"yattrs\" }"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
})
}))
s.Wait(0)
// This should reorder the ops before sending to the server.
// We put the delete last to ensure that the xattr order is preserved.
mutateOps := []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagMkDirP,
Path: "xatest.test",
Value: []byte("\"test value\""),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath,
Path: "xatest.ytest",
Value: []byte("\"test value2\""),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath,
Path: "xatest.ztest",
Value: []byte("\"test valuez\""),
},
{
Op: memd.SubDocOpDelete,
Flags: memd.SubdocFlagXattrPath,
Path: "xatest.ztest",
},
}
s.PushOp(agent.MutateIn(MutateInOptions{
Key: []byte("testXattrReorder"),
Ops: mutateOps,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *MutateInResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Mutate operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
if len(res.Ops) != 5 {
s.Fatalf("MutateIn operation wrong count was %d", len(res.Ops))
}
if res.Ops[0].Err != nil {
s.Fatalf("MutateIn operation 1 failed: %v", res.Ops[0].Err)
}
if res.Ops[1].Err != nil {
s.Fatalf("MutateIn operation 2 failed: %v", res.Ops[1].Err)
}
if res.Ops[2].Err != nil {
s.Fatalf("MutateIn operation 3 failed: %v", res.Ops[2].Err)
}
if res.Ops[3].Err != nil {
s.Fatalf("MutateIn operation 4 failed: %v", res.Ops[2].Err)
}
if res.Ops[4].Err != nil {
s.Fatalf("MutateIn operation 5 failed: %v", res.Ops[2].Err)
}
})
}))
s.Wait(0)
lookupOps := []SubDocOp{
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagXattrPath,
Path: "xatest.test",
},
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagNone,
Path: "x",
},
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagXattrPath,
Path: "xatest.ytest",
},
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagXattrPath,
Path: "xatest.ztest",
},
}
s.PushOp(agent.LookupIn(LookupInOptions{
Key: []byte("testXattrReorder"),
Ops: lookupOps,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *LookupInResult, err error) {
s.Wrap(func() {
if len(res.Ops) != 4 {
s.Fatalf("Lookup operation wrong count: %d", len(res.Ops))
}
if res.Ops[0].Err != nil {
s.Fatalf("Lookup operation 1 failed: %v", res.Ops[0].Err)
}
if res.Ops[1].Err != nil {
s.Fatalf("Lookup operation 2 failed: %v", res.Ops[1].Err)
}
if res.Ops[2].Err != nil {
s.Fatalf("Lookup operation 3 failed: %v", res.Ops[2].Err)
}
if res.Ops[3].Err == nil {
s.Fatalf("Lookup operation 4 should have failed")
}
if !bytes.Equal(res.Ops[0].Value, []byte(`"test value"`)) {
s.Fatalf("Unexpected xatest.test value %s", res.Ops[0].Value)
}
if !bytes.Equal(res.Ops[1].Value, []byte(`"x value"`)) {
s.Fatalf("Unexpected document value %s", res.Ops[1].Value)
}
if !bytes.Equal(res.Ops[2].Value, []byte(`"test value2"`)) {
s.Fatalf("Unexpected xatest.ytest value %s", res.Ops[2].Value)
}
})
}))
s.Wait(0)
}
// Create As Deleted
func (suite *StandardTestSuite) TestTombstones() {
suite.EnsureSupportsFeature(TestFeatureCreateDeleted)
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.MutateIn(MutateInOptions{
Key: []byte("TestInsertTombstoneWithXattr"),
Flags: memd.SubdocDocFlagCreateAsDeleted | memd.SubdocDocFlagAccessDeleted | memd.SubdocDocFlagMkDoc,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Value: []byte("{\"test\":\"test\"}"),
Path: "test",
Flags: memd.SubdocFlagXattrPath,
},
},
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *MutateInResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.LookupIn(LookupInOptions{
Key: []byte("TestInsertTombstoneWithXattr"),
Flags: memd.SubdocDocFlagAccessDeleted,
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "test",
Flags: memd.SubdocFlagXattrPath,
},
},
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *LookupInResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
if len(res.Ops) != 1 {
s.Fatalf("LookupIn operation wrong count was %d", len(res.Ops))
}
if res.Ops[0].Err != nil {
s.Fatalf("LookupIn operation failed: %v", res.Ops[0].Err)
}
if len(res.Ops[0].Value) == 0 {
s.Fatalf("LookupIn operation returned no value")
}
if !res.Internal.IsDeleted {
s.Fatalf("LookupIn operation should have return IDeleted==true")
}
})
}))
s.Wait(0)
s.PushOp(agent.LookupIn(LookupInOptions{
Key: []byte("TestInsertTombstoneWithXattr"),
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "test",
Flags: memd.SubdocFlagXattrPath,
},
},
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *LookupInResult, err error) {
s.Wrap(func() {
if err == nil {
s.Fatalf("Get operation should have failed")
}
})
}))
s.Wait(0)
}
func (suite *StandardTestSuite) TestReplaceBodyWithXattr() {
suite.EnsureSupportsFeature(TestFeatureReplaceBodyWithXattr)
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testReplaceBodyWithXattr"),
Value: []byte("{\"mybody\":\"isnotxattrs\"}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.MutateIn(MutateInOptions{
Key: []byte("testReplaceBodyWithXattr"),
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "mybodyafterward",
Value: []byte("{\"mybody\":\"willbethexattrafterwardthough\"}"),
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagMkDirP,
},
},
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *MutateInResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("MutateIn operation failed %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.MutateIn(MutateInOptions{
Key: []byte("testReplaceBodyWithXattr"),
Ops: []SubDocOp{
{
Op: memd.SubDocOpReplaceBodyWithXattr,
Path: "mybodyafterward",
Flags: memd.SubdocFlagXattrPath,
},
},
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *MutateInResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("MutateIn operation failed %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.Get(GetOptions{
Key: []byte("testReplaceBodyWithXattr"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get operation failed %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
var body map[string]string
err = json.Unmarshal(res.Value, &body)
if err != nil {
s.Fatalf("Unmarshal failed %v", err)
}
if len(body) != 1 {
s.Fatalf("Expected body contain one key, was %v", body)
}
val, ok := body["mybody"]
if !ok {
s.Fatalf("Expected body contain mybody, was %v", body)
}
if val != "willbethexattrafterwardthough" {
s.Fatalf("Expected mybody value to be willbethexattrafterwardthough was %v", val)
}
})
}))
s.Wait(0)
}
func (suite *StandardTestSuite) TestMutateInNoOps() {
agent := suite.DefaultAgent()
_, err := agent.MutateIn(MutateInOptions{
Key: []byte("TestMutateInNoOps"),
}, func(result *MutateInResult, err error) {
})
if !errors.Is(err, ErrInvalidArgument) {
suite.T().Fatalf("Expected invalid argument error but was %v", err)
}
}
func (suite *StandardTestSuite) TestMutateInInsertString() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinInsertString")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{})
expectedVal := suite.mustMarshal("bar2")
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictAdd,
Path: "foo2",
Value: expectedVal,
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, _ := suite.mustGetDoc(agent, s, key)
var target map[string]json.RawMessage
err = json.Unmarshal(val, &target)
suite.Require().Nil(err)
suite.Require().Contains(target, "foo2")
if !bytes.Equal(expectedVal, target["foo2"]) {
suite.T().Fatalf("Expected value to be %v but was %v", string(target["foo2"]), string(val))
}
}
func (suite *StandardTestSuite) TestMutateInRemove() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinRemove")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": "bar",
})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDelete,
Path: "foo",
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, _ := suite.mustGetDoc(agent, s, key)
var target map[string]json.RawMessage
err = json.Unmarshal(val, &target)
suite.Require().Nil(err)
suite.Require().NotContains(target, "foo2")
}
func (suite *StandardTestSuite) TestMutateInInsertStringExists() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinInsertStringExists")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": "bar",
})
expectedVal := suite.mustMarshal("bar2")
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictAdd,
Path: "foo",
Value: expectedVal,
},
}, key, cas, 0)
if !errors.Is(err, ErrPathExists) {
suite.T().Fatalf("Expected error to be %v but was %v", ErrPathExists, err)
}
}
func (suite *StandardTestSuite) TestMutateInReplaceString() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinReplaceString")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": "bar",
})
expectedVal := suite.mustMarshal("bar2")
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpReplace,
Path: "foo",
Value: expectedVal,
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, _ := suite.mustGetDoc(agent, s, key)
var target map[string]json.RawMessage
err = json.Unmarshal(val, &target)
suite.Require().Nil(err)
suite.Require().Contains(target, "foo")
if !bytes.Equal(expectedVal, target["foo"]) {
suite.T().Fatalf("Expected value to be %v but was %v", string(target["foo2"]), string(val))
}
}
func (suite *StandardTestSuite) TestMutateInReplaceFullDoc() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinReplaceFullDoc")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": "bar",
})
expectedVal := suite.mustMarshal(map[string]interface{}{
"foo2": "bar2",
})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpSetDoc,
Path: "",
Value: expectedVal,
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, _ := suite.mustGetDoc(agent, s, key)
if !bytes.Equal(expectedVal, val) {
suite.T().Fatalf("Expected value to be %v but was %v", string(expectedVal), string(val))
}
}
func (suite *StandardTestSuite) TestMutateInReplaceStringDoesntExist() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinReplaceStringDoesntExist")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{})
expectedVal := suite.mustMarshal("bar2")
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpReplace,
Path: "foo",
Value: expectedVal,
},
}, key, cas, 0)
if !errors.Is(err, ErrPathNotFound) {
suite.T().Fatalf("Expected error to be %v but was %v", ErrPathNotFound, err)
}
}
func (suite *StandardTestSuite) TestMutateInSetString() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinSetString")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": "bar",
})
expectedVal := suite.mustMarshal("bar2")
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "foo",
Value: expectedVal,
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, _ := suite.mustGetDoc(agent, s, key)
var target map[string]json.RawMessage
err = json.Unmarshal(val, &target)
suite.Require().Nil(err)
suite.Require().Contains(target, "foo")
if !bytes.Equal(expectedVal, target["foo"]) {
suite.T().Fatalf("Expected value to be %v but was %v", string(target["foo2"]), string(val))
}
}
func (suite *StandardTestSuite) TestMutateInSetStringDoesNotExist() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinSetStringDoesNotExist")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{})
expectedVal := suite.mustMarshal("bar")
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "foo",
Value: expectedVal,
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, _ := suite.mustGetDoc(agent, s, key)
var target map[string]json.RawMessage
err = json.Unmarshal(val, &target)
suite.Require().Nil(err)
suite.Require().Contains(target, "foo")
if !bytes.Equal(expectedVal, target["foo"]) {
suite.T().Fatalf("Expected value to be %v but was %v", string(target["foo2"]), string(val))
}
}
func (suite *StandardTestSuite) TestMutateInArrayAppend() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinArrayAppend")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": []string{"hello"},
})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpArrayPushLast,
Path: "foo",
Value: suite.mustMarshal("world"),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, _ := suite.mustGetDoc(agent, s, key)
var target map[string]json.RawMessage
err = json.Unmarshal(val, &target)
suite.Require().Nil(err)
expectedVal := suite.mustMarshal([]string{"hello", "world"})
suite.Require().Contains(target, "foo")
if !bytes.Equal(expectedVal, target["foo"]) {
suite.T().Fatalf("Expected value to be %v but was %v", string(expectedVal), string(target["foo"]))
}
}
func (suite *StandardTestSuite) TestMutateInArrayAddUnique() {
if suite.IsMockServer() {
suite.T().Skip("Test skipped due to mock bug")
}
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinArrayAddUnique")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": []string{"hello", "world"},
})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpArrayAddUnique,
Path: "foo",
Value: suite.mustMarshal("cruel"),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, _ := suite.mustGetDoc(agent, s, key)
var target map[string]json.RawMessage
err = json.Unmarshal(val, &target)
suite.Require().Nil(err)
expectedVal := suite.mustMarshal([]string{"hello", "world", "cruel"})
suite.Require().Contains(target, "foo")
if !bytes.Equal(expectedVal, target["foo"]) {
suite.T().Fatalf("Expected value to be %v but was %v", string(expectedVal), string(target["foo"]))
}
}
func (suite *StandardTestSuite) TestMutateInArrayAddUniqueAlreadyExists() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinArrayAddUniqueAlreadyExist")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": []string{"hello", "world", "cruel"},
})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpArrayAddUnique,
Path: "foo",
Value: suite.mustMarshal("cruel"),
},
}, key, cas, 0)
if !errors.Is(err, ErrPathExists) {
suite.T().Fatalf("Expected error to be %v but was %v", ErrPathNotFound, err)
}
}
func (suite *StandardTestSuite) TestMutateInCounterIncrement() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinCounterIncrement")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": 10,
})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpCounter,
Path: "foo",
Value: suite.mustMarshal(5),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, _ := suite.mustGetDoc(agent, s, key)
var target map[string]json.RawMessage
err = json.Unmarshal(val, &target)
suite.Require().Nil(err)
expectedVal := suite.mustMarshal(15)
suite.Require().Contains(target, "foo")
if !bytes.Equal(expectedVal, target["foo"]) {
suite.T().Fatalf("Expected value to be %v but was %v", string(expectedVal), string(target["foo"]))
}
}
func (suite *StandardTestSuite) TestMutateInCounterDecrement() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinCounterDecrement")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": 10,
})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpCounter,
Path: "foo",
Value: suite.mustMarshal(-5),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, _ := suite.mustGetDoc(agent, s, key)
var target map[string]json.RawMessage
err = json.Unmarshal(val, &target)
suite.Require().Nil(err)
expectedVal := suite.mustMarshal(5)
suite.Require().Contains(target, "foo")
if !bytes.Equal(expectedVal, target["foo"]) {
suite.T().Fatalf("Expected value to be %v but was %v", string(expectedVal), string(target["foo"]))
}
}
func (suite *StandardTestSuite) TestMutateInRemoveXattr() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinRemoveXAttr")
cas, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: []byte("\"bar\""),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}, key, 0, memd.SubdocDocFlagMkDoc)
suite.Require().Nil(err, err)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDelete,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
if !errors.Is(val.Ops[0].Err, ErrPathNotFound) {
suite.T().Fatalf("Expected error to be %v but was %v", ErrPathNotFound, err)
}
}
func (suite *StandardTestSuite) TestMutateInRemoveXattrDoesNotExist() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinRemoveXAttrNotExist")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDelete,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
},
}, key, cas, 0)
if !errors.Is(err, ErrPathNotFound) {
suite.T().Fatalf("Expected error to be %v but was %v", ErrPathNotFound, err)
}
}
func (suite *StandardTestSuite) TestMutateInInsertXattrExists() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinInsertXAttrExists")
cas, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: []byte("\"bar\""),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}, key, 0, memd.SubdocDocFlagMkDoc)
suite.Require().Nil(err, err)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictAdd,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: []byte("\"bar\""),
},
}, key, cas, 0)
if !errors.Is(err, ErrPathExists) {
suite.T().Fatalf("Expected error to be %v but was %v", ErrPathExists, err)
}
}
func (suite *StandardTestSuite) TestMutateInReplaceXattr() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinReplaceXAttr")
cas, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: []byte("\"bar\""),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}, key, 0, memd.SubdocDocFlagMkDoc)
suite.Require().Nil(err, err)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpReplace,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
Value: []byte("\"bar2\""),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
res, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
if !bytes.Equal([]byte("\"bar2\""), res.Ops[0].Value) {
suite.T().Fatalf("Expected value to be %v but was %v", "\"bar2\"", string(res.Ops[0].Value))
}
}
func (suite *StandardTestSuite) TestMutateInReplaceXattrNotExist() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinReplaceXAttrNotExist")
cas, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}, key, 0, memd.SubdocDocFlagMkDoc)
suite.Require().Nil(err, err)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpReplace,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
Value: []byte("\"bar2\""),
},
}, key, cas, 0)
if !errors.Is(err, ErrPathNotFound) {
suite.T().Fatalf("Expected error to be %v but was %v", ErrPathNotFound, err)
}
}
func (suite *StandardTestSuite) TestMutateInSetXattr() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinSetXattr")
cas, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: []byte("\"bar\""),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}, key, 0, memd.SubdocDocFlagMkDoc)
suite.Require().Nil(err, err)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
Value: []byte("\"bar2\""),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
res, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
if !bytes.Equal([]byte("\"bar2\""), res.Ops[0].Value) {
suite.T().Fatalf("Expected value to be %v but was %v", "\"bar2\"", string(res.Ops[0].Value))
}
}
func (suite *StandardTestSuite) TestMutateInSetXattrNotExist() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinSetXAttrNotExist")
cas, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}, key, 0, memd.SubdocDocFlagMkDoc)
suite.Require().Nil(err, err)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
Value: []byte("\"bar2\""),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
res, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
if !bytes.Equal([]byte("\"bar2\""), res.Ops[0].Value) {
suite.T().Fatalf("Expected value to be %v but was %v", "\"bar2\"", string(res.Ops[0].Value))
}
}
func (suite *StandardTestSuite) TestMutateInArrayAppendXattr() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinArrayAppendXattr")
cas, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: suite.mustMarshal([]string{"hello"}),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}, key, 0, memd.SubdocDocFlagMkDoc)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpArrayPushLast,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: suite.mustMarshal("world"),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
expectedVal := suite.mustMarshal([]string{"hello", "world"})
if !bytes.Equal(expectedVal, val.Ops[0].Value) {
suite.T().Fatalf("Expected value to be %v but was %v", string(expectedVal), string(val.Ops[0].Value))
}
}
func (suite *StandardTestSuite) TestMutateInArrayAddUniqueXattr() {
if suite.IsMockServer() {
suite.T().Skip("Test skipped due to mock bug")
}
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinArrayAddUniqueXattr")
cas, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: suite.mustMarshal([]string{"hello", "world"}),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}, key, 0, memd.SubdocDocFlagMkDoc)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpArrayAddUnique,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: suite.mustMarshal("cruel"),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
expectedVal := suite.mustMarshal([]string{"hello", "world", "cruel"})
if !bytes.Equal(expectedVal, val.Ops[0].Value) {
suite.T().Fatalf("Expected value to be %v but was %v", string(expectedVal), string(val.Ops[0].Value))
}
}
func (suite *StandardTestSuite) TestMutateInArrayAddUniqueAlreadyExistsXattr() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinArrayAddUniqueAlreadyExistXattr")
cas, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: suite.mustMarshal([]string{"hello", "world", "cruel"}),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}, key, 0, memd.SubdocDocFlagMkDoc)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpArrayAddUnique,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: suite.mustMarshal("cruel"),
},
}, key, cas, 0)
if !errors.Is(err, ErrPathExists) {
suite.T().Fatalf("Expected error to be %v but was %v", ErrPathNotFound, err)
}
}
func (suite *StandardTestSuite) TestMutateInCounterIncrementXattr() {
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinCounterIncrementXattr")
cas, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: suite.mustMarshal(10),
},
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
}, key, 0, memd.SubdocDocFlagMkDoc)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpCounter,
Flags: memd.SubdocFlagXattrPath,
Path: "foo",
Value: suite.mustMarshal(5),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
expectedVal := suite.mustMarshal(15)
if !bytes.Equal(expectedVal, val.Ops[0].Value) {
suite.T().Fatalf("Expected value to be %v but was %v", string(expectedVal), string(val.Ops[0].Value))
}
}
func (suite *StandardTestSuite) TestMutateInExpandMacroCas() {
suite.EnsureSupportsFeature(TestFeatureExpandMacros)
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinExpandMacroCas")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "foo",
Value: suite.mustMarshal("${Mutation.CAS}"),
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "foo",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
var resultCas string
err = json.Unmarshal(val.Ops[0].Value, &resultCas)
suite.Require().Nil(err, err)
// We should improve this to check the actual cas value.
suite.Require().NotEqual("${Mutation.CAS}", resultCas)
}
func (suite *StandardTestSuite) TestMutateInExpandMacroCRC32() {
suite.EnsureSupportsFeature(TestFeatureExpandMacros)
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinExpandMacroCRC32")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": "bar",
})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "xfoo",
Value: suite.mustMarshal("${Mutation.value_crc32c}"),
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
},
{
Op: memd.SubDocOpDictSet,
Path: "foo",
Value: suite.mustMarshal("value"),
},
}, key, cas, 0)
suite.Require().Nil(err, err)
// We need to do 2 ops because older server versions only allow a single xattr lookup.
val, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "xfoo",
Flags: memd.SubdocFlagXattrPath,
},
{
Op: memd.SubDocOpGet,
Path: "foo",
},
}, key)
suite.Require().Nil(err, err)
suite.Require().Nil(val.Ops[0].Err, val.Ops[0].Err)
documentVal, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "$document",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
suite.Require().Nil(documentVal.Ops[0].Err, documentVal.Ops[0].Err)
var resultCRC string
err = json.Unmarshal(val.Ops[0].Value, &resultCRC)
suite.Require().Nil(err, err)
// We pull the actual crc value from the doc metadata.
crcStruct := struct {
CRC32 string `json:"value_crc32c,omitempty"`
}{}
err = json.Unmarshal(documentVal.Ops[0].Value, &crcStruct)
suite.Require().Nil(err, err)
suite.Require().Equal(crcStruct.CRC32, resultCRC)
}
func (suite *StandardTestSuite) TestMutateInExpandMacroSeqNo() {
suite.EnsureSupportsFeature(TestFeatureExpandMacros)
suite.EnsureSupportsFeature(TestFeatureExpandMacrosSeqNo)
agent, s := suite.GetAgentAndHarness()
key := []byte("mutateinExpandMacroSeqNo")
cas := suite.mustSetDoc(agent, s, key, map[string]interface{}{
"foo": "bar",
})
_, err := suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "xfoo",
Value: suite.mustMarshal("${Mutation.seqno}"),
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
},
}, key, cas, 0)
suite.Require().Nil(err, err)
val, err := suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "xfoo",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
suite.Require().Nil(val.Ops[0].Err, val.Ops[0].Err)
var seqno string
err = json.Unmarshal(val.Ops[0].Value, &seqno)
suite.Require().Nil(err, err)
suite.Require().NotZero(seqno)
_, err = suite.mutateIn(agent, s, []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "xfoo",
Value: suite.mustMarshal("${Mutation.seqno}"),
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
},
}, key, val.Cas, 0)
suite.Require().Nil(err, err)
val, err = suite.lookupDoc(agent, s, []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "xfoo",
Flags: memd.SubdocFlagXattrPath,
},
}, key)
suite.Require().Nil(err, err)
suite.Require().Nil(val.Ops[0].Err, val.Ops[0].Err)
var seqno2 string
err = json.Unmarshal(val.Ops[0].Value, &seqno2)
suite.Require().Nil(err, err)
suite.Require().NotZero(seqno2)
// We test that performing 2 mutations creates a sequential sequence number.
seqnoInt, err := strconv.ParseInt(strings.Replace(seqno, "0x", "", -1), 16, 64)
suite.Require().Nil(err, err)
seqno2Int, err := strconv.ParseInt(strings.Replace(seqno2, "0x", "", -1), 16, 64)
suite.Require().Nil(err, err)
suite.Require().Greater(seqno2Int, seqnoInt)
}
func (suite *StandardTestSuite) TestPreserveExpiryMutateIn() {
suite.EnsureSupportsFeature(TestFeaturePreserveExpiry)
agent, s := suite.GetAgentAndHarness()
expiry := uint32(25)
// Set
s.PushOp(agent.Set(SetOptions{
Key: []byte("testmutateinpreserveExpiry"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
Expiry: expiry,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
ops := []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Value: []byte("{}"),
Path: "test",
},
}
s.PushOp(agent.MutateIn(MutateInOptions{
Key: []byte("testmutateinpreserveExpiry"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
PreserveExpiry: true,
Ops: ops,
}, func(res *MutateInResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Mutatein operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
// Get
s.PushOp(agent.GetMeta(GetMetaOptions{
Key: []byte("testmutateinpreserveExpiry"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetMetaResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("GetMeta operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
expectedExpiry := uint32(time.Now().Unix() + int64(expiry-5))
if res.Expiry < expectedExpiry {
s.Fatalf("Invalid expiry received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(3, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "testmutateinpreserveExpiry")
suite.AssertOpSpan(nilParents[1], "MutateIn", agent.BucketName(), memd.CmdSubDocMultiMutation.Name(), 1, false, "testmutateinpreserveExpiry")
suite.AssertOpSpan(nilParents[2], "GetMeta", agent.BucketName(), memd.CmdGetMeta.Name(), 1, false, "testmutateinpreserveExpiry")
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "MutateIn", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "GetMeta", 1, false, false)
}
func (suite *StandardTestSuite) TestSubdocCasMismatch() {
agent, s := suite.GetAgentAndHarness()
s.PushOp(agent.Set(SetOptions{
Key: []byte("testSubdocCasMismatch"),
Value: []byte("{\"x\":\"xattrs\", \"y\":\"yattrs\" }"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
})
}))
s.Wait(0)
s.PushOp(agent.MutateIn(MutateInOptions{
Key: []byte("testSubdocCasMismatch"),
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
},
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
Cas: 1234,
}, func(res *MutateInResult, err error) {
s.Wrap(func() {
if !errors.Is(err, ErrCasMismatch) {
s.Fatalf("Mutate operation should have failed with Cas Mismatch but was: %v", err)
}
})
}))
s.Wait(0)
// With Add flag
s.PushOp(agent.MutateIn(MutateInOptions{
Key: []byte("testSubdocCasMismatch"),
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagNone,
Path: "x",
Value: []byte("\"x value\""),
},
},
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
Flags: memd.SubdocDocFlagAddDoc,
}, func(res *MutateInResult, err error) {
s.Wrap(func() {
if !errors.Is(err, ErrDocumentExists) {
s.Fatalf("Mutate operation should have failed with Exists but was: %v", err)
}
})
}))
s.Wait(0)
}
gocbcore-10.2.3/crudcomponent_test.go 0000664 0000000 0000000 00000015404 14417540156 0017641 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"github.com/google/uuid"
"github.com/couchbase/gocbcore/v10/memd"
)
func (suite *StandardTestSuite) TestResourceUnits() {
suite.EnsureSupportsFeature(TestFeatureResourceUnits)
agent, s := suite.GetAgentAndHarness()
docID := uuid.NewString()
var resourceUnits *ResourceUnitResult
s.PushOp(agent.Set(SetOptions{
Key: []byte(docID),
Value: []byte("{\"x\":\"xattrs\"}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
resourceUnits = res.Internal.ResourceUnits
})
}))
s.Wait(0)
if suite.Assert().NotNil(resourceUnits) {
suite.Require().GreaterOrEqual(1, int(resourceUnits.WriteUnits))
}
s.PushOp(agent.Get(GetOptions{
Key: []byte(docID),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
resourceUnits = res.Internal.ResourceUnits
})
}))
s.Wait(0)
if suite.Assert().NotNil(resourceUnits) {
suite.Require().GreaterOrEqual(1, int(resourceUnits.ReadUnits))
}
s.PushOp(agent.Touch(TouchOptions{
Key: []byte(docID),
Expiry: 5,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *TouchResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Get operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
resourceUnits = res.Internal.ResourceUnits
})
}))
s.Wait(0)
if suite.Assert().NotNil(resourceUnits) {
suite.Require().GreaterOrEqual(1, int(resourceUnits.ReadUnits))
suite.Require().GreaterOrEqual(1, int(resourceUnits.WriteUnits))
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(3, len(nilParents)) {
suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, docID)
suite.AssertOpSpan(nilParents[1], "Get", agent.BucketName(), memd.CmdGet.Name(), 1, false, docID)
suite.AssertOpSpan(nilParents[2], "Touch", agent.BucketName(), memd.CmdTouch.Name(), 1, false, docID)
}
}
suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Get", 1, false, false)
suite.VerifyKVMetrics(suite.meter, "Touch", 1, false, false)
}
// At time of writing compute units were not applied for a failed unlock.
// func (suite *StandardTestSuite) TestResourceUnitsLockedRetries() {
// suite.EnsureSupportsFeature(TestFeatureResourceUnits)
//
// agent, s := suite.GetAgentAndHarness()
//
// docID := uuid.NewString()
//
// var resourceUnits *ResourceUnitResult
// s.PushOp(agent.Set(SetOptions{
// Key: []byte(docID),
// Value: []byte("{\"x\":\"xattrs\"}"),
// CollectionName: suite.CollectionName,
// ScopeName: suite.ScopeName,
// }, func(res *StoreResult, err error) {
// s.Wrap(func() {
// if err != nil {
// s.Fatalf("Set operation failed: %v", err)
// }
//
// resourceUnits = res.Internal.ResourceUnits
// })
// }))
// s.Wait(0)
//
// s.PushOp(agent.GetAndLock(GetAndLockOptions{
// Key: []byte(docID),
// LockTime: 2,
// CollectionName: suite.CollectionName,
// ScopeName: suite.ScopeName,
// }, func(res *GetAndLockResult, err error) {
// s.Wrap(func() {
// if err != nil {
// s.Fatalf("Get operation failed: %v", err)
// }
// if res.Cas == Cas(0) {
// s.Fatalf("Invalid cas received")
// }
//
// resourceUnits = res.Internal.ResourceUnits
// })
// }))
// s.Wait(0)
//
// if suite.Assert().NotNil(resourceUnits) {
// suite.Require().GreaterOrEqual(1, int(resourceUnits.ReadUnits))
// suite.Require().GreaterOrEqual(1, int(resourceUnits.WriteUnits))
// }
//
// s.PushOp(agent.GetAndLock(GetAndLockOptions{
// Key: []byte(docID),
// CollectionName: suite.CollectionName,
// ScopeName: suite.ScopeName,
// RetryStrategy: NewBestEffortRetryStrategy(ControlledBackoff),
// }, func(res *GetAndLockResult, err error) {
// s.Wrap(func() {
// if err != nil {
// s.Fatalf("Get operation failed: %v", err)
// }
// if res.Cas == Cas(0) {
// s.Fatalf("Invalid cas received")
// }
//
// resourceUnits = res.Internal.ResourceUnits
// })
// }))
// s.Wait(5)
//
// if suite.Assert().NotNil(resourceUnits) {
// suite.Require().GreaterOrEqual(1, int(resourceUnits.ReadUnits))
// suite.Require().GreaterOrEqual(1, int(resourceUnits.WriteUnits))
// }
//
// if suite.Assert().Contains(suite.tracer.Spans, nil) {
// nilParents := suite.tracer.Spans[nil]
// if suite.Assert().Equal(3, len(nilParents)) {
// suite.AssertOpSpan(nilParents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, docID)
// suite.AssertOpSpan(nilParents[1], "GetAndLock", agent.BucketName(), memd.CmdGetLocked.Name(), 1, true, docID)
// suite.AssertOpSpan(nilParents[2], "GetAndLock", agent.BucketName(), memd.CmdGetLocked.Name(), 2, true, docID)
//
// if suite.Assert().NotNil(resourceUnits) {
// numReqs := len(nilParents[2].Spans[memd.CmdGetLocked.Name()])
// suite.Assert().Equal(numReqs, int(resourceUnits.ReadUnits))
// suite.Assert().Equal(numReqs, int(resourceUnits.WriteUnits))
// }
// }
// }
//
// suite.VerifyKVMetrics(suite.meter, "Set", 1, false, false)
// suite.VerifyKVMetrics(suite.meter, "GetAndLock", 3, true, false)
// }
// At time of writing compute units were not supported for get collection ID.
// func (suite *StandardTestSuite) TestResourceUnitsCollectionUnknown() {
// suite.EnsureSupportsFeature(TestFeatureResourceUnits)
//
// agent, s := suite.GetAgentAndHarness()
//
// colName := uuid.NewString()
// _, err := testCreateCollection(colName, globalTestConfig.ScopeName, globalTestConfig.BucketName, agent)
// suite.Require().Nil(err)
//
// // Who knows what's happened during creating the collection.
// suite.meter.Reset()
// suite.tracer.Reset()
//
// docID := uuid.NewString()
//
// var resourceUnits *ResourceUnitResult
// s.PushOp(agent.Set(SetOptions{
// Key: []byte(docID),
// Value: []byte("{\"x\":\"xattrs\"}"),
// CollectionName: colName,
// ScopeName: suite.ScopeName,
// }, func(res *StoreResult, err error) {
// s.Wrap(func() {
// if err != nil {
// s.Fatalf("Set operation failed: %v", err)
// }
//
// resourceUnits = res.Internal.ResourceUnits
// })
// }))
// s.Wait(0)
//
// if suite.Assert().NotNil(resourceUnits) {
// suite.Require().GreaterOrEqual(1, int(resourceUnits.ReadUnits))
// suite.Require().GreaterOrEqual(1, int(resourceUnits.WriteUnits))
// }
// }
gocbcore-10.2.3/curdcomponent_crud_bench_test.go 0000664 0000000 0000000 00000005501 14417540156 0022012 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"fmt"
"github.com/google/uuid"
"sync/atomic"
"testing"
)
func BenchmarkSet(b *testing.B) {
b.ReportAllocs()
suite := GetBenchSuite()
agent := suite.GetAgent()
key := []byte(uuid.New().String())
// Generate 256 bytes of random data for the document
randomBytes := make([]byte, 256)
for i := 0; i < len(randomBytes); i++ {
randomBytes[i] = byte(i)
}
var i uint32
suite.RunParallel(b, func(cb func(error)) error {
keyNum := atomic.AddUint32(&i, 1)
_, err := agent.Set(SetOptions{
Key: []byte(fmt.Sprintf("%s-%d", key, keyNum)),
Value: randomBytes,
}, func(result *StoreResult, err error) {
cb(err)
})
return err
})
}
func BenchmarkReplace(b *testing.B) {
b.ReportAllocs()
suite := GetBenchSuite()
agent, s := suite.GetAgentAndHarness(b)
key := []byte(uuid.New().String())
s.PushOp(agent.Set(SetOptions{
Key: key,
Value: []byte("{}"),
}, func(result *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Failed to set: %v", err)
}
})
}))
s.Wait(0)
// Generate 256 bytes of random data for the document
randomBytes := make([]byte, 256)
for i := 0; i < len(randomBytes); i++ {
randomBytes[i] = byte(i)
}
suite.RunParallel(b, func(cb func(error)) error {
_, err := agent.Replace(ReplaceOptions{
Key: key,
Value: randomBytes,
}, func(result *StoreResult, err error) {
cb(err)
})
return err
})
}
func BenchmarkGet(b *testing.B) {
b.ReportAllocs()
suite := GetBenchSuite()
agent, s := suite.GetAgentAndHarness(b)
key := []byte(uuid.New().String())
// Generate 256 bytes of random data for the document
randomBytes := make([]byte, 256)
for i := 0; i < len(randomBytes); i++ {
randomBytes[i] = byte(i)
}
s.PushOp(agent.Set(SetOptions{
Key: key,
Value: randomBytes,
}, func(result *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Failed to set: %v", err)
}
})
}))
s.Wait(0)
suite.RunParallel(b, func(cb func(error)) error {
_, err := agent.Get(GetOptions{
Key: key,
}, func(result *GetResult, err error) {
cb(err)
})
return err
})
}
func BenchmarkSetGet(b *testing.B) {
b.ReportAllocs()
suite := GetBenchSuite()
agent := suite.GetAgent()
key := []byte(uuid.New().String())
// Generate 256 bytes of random data for the document
randomBytes := make([]byte, 256)
for i := 0; i < len(randomBytes); i++ {
randomBytes[i] = byte(i)
}
runGet := func(cb func(error)) {
_, err := agent.Get(GetOptions{
Key: key,
}, func(result *GetResult, err error) {
cb(err)
})
if err != nil {
cb(err)
}
}
suite.RunParallel(b, func(cb func(error)) error {
_, err := agent.Set(SetOptions{
Key: key,
Value: randomBytes,
}, func(result *StoreResult, err error) {
if err != nil {
cb(err)
return
}
runGet(cb)
})
return err
})
}
gocbcore-10.2.3/dcp.go 0000664 0000000 0000000 00000015015 14417540156 0014466 0 ustar 00root root 0000000 0000000 package gocbcore
// OpenStreamFilterOptions are the filtering options available to the OpenStream operation.
type OpenStreamFilterOptions struct {
ScopeID uint32
CollectionIDs []uint32
}
// OpenStreamStreamOptions are the stream options available to the OpenStream operation.
type OpenStreamStreamOptions struct {
StreamID uint16
}
// OpenStreamManifestOptions are the manifest options available to the OpenStream operation.
type OpenStreamManifestOptions struct {
ManifestUID uint64
}
// OpenStreamOptions are the options available to the OpenStream operation.
type OpenStreamOptions struct {
FilterOptions *OpenStreamFilterOptions
StreamOptions *OpenStreamStreamOptions
ManifestOptions *OpenStreamManifestOptions
}
// GetVbucketSeqnoFilterOptions are the filter options available to the GetVbucketSeqno operation.
type GetVbucketSeqnoFilterOptions struct {
CollectionID uint32
}
// GetVbucketSeqnoOptions are the options available to the GetVbucketSeqno operation.
type GetVbucketSeqnoOptions struct {
FilterOptions *GetVbucketSeqnoFilterOptions
}
// CloseStreamStreamOptions are the stream options available to the CloseStream operation.
type CloseStreamStreamOptions struct {
StreamID uint16
}
// CloseStreamOptions are the options available to the CloseStream operation.
type CloseStreamOptions struct {
StreamOptions *CloseStreamStreamOptions
}
// SnapshotState represents the state of a particular cluster snapshot.
type SnapshotState uint32
// HasInMemory returns whether this snapshot is available in memory.
func (s SnapshotState) HasInMemory() bool {
return uint32(s)&1 != 0
}
// HasOnDisk returns whether this snapshot is available on disk.
func (s SnapshotState) HasOnDisk() bool {
return uint32(s)&2 != 0
}
// FailoverEntry represents a single entry in the server fail-over log.
type FailoverEntry struct {
VbUUID VbUUID
SeqNo SeqNo
}
// DcpSnapshotMarker represents a single response from the server
type DcpSnapshotMarker struct {
StartSeqNo, EndSeqNo uint64
VbID, StreamID uint16
SnapshotType SnapshotState
MaxVisibleSeqNo, HighCompletedSeqNo, SnapshotTimeStamp uint64
}
// DcpMutation represents a single DCP mutation from the server
type DcpMutation struct {
SeqNo, RevNo uint64
Cas uint64
Flags, Expiry, LockTime uint32
CollectionID uint32
VbID uint16
StreamID uint16
Datatype uint8
Key, Value []byte
}
// DcpDeletion represents a single DCP deletion from the server
type DcpDeletion struct {
SeqNo, RevNo uint64
Cas uint64
DeleteTime uint32
CollectionID uint32
VbID uint16
StreamID uint16
Datatype uint8
Key, Value []byte
}
// DcpExpiration represents a single DCP expiration from the server
type DcpExpiration struct {
SeqNo, RevNo uint64
Cas uint64
DeleteTime uint32
CollectionID uint32
VbID uint16
StreamID uint16
Key []byte
}
// DcpCollectionCreation represents a collection create DCP event from the server
type DcpCollectionCreation struct {
SeqNo uint64
Version uint8
VbID uint16
ManifestUID uint64
ScopeID uint32
CollectionID uint32
Ttl uint32
StreamID uint16
Key []byte
}
// DcpCollectionDeleteion represents a collection delete DCP event from the server
type DcpCollectionDeletion struct {
SeqNo uint64
ManifestUID uint64
ScopeID uint32
CollectionID uint32
StreamID uint16
VbID uint16
Version uint8
}
// DcpCollectionFlush represents a collection flush DCP event from the server
type DcpCollectionFlush struct {
SeqNo uint64
Version uint8
VbID uint16
ManifestUID uint64
CollectionID uint32
StreamID uint16
}
// DcpScopeCreation represents a scope creation DCP event from the server
type DcpScopeCreation struct {
SeqNo uint64
Version uint8
VbID uint16
ManifestUID uint64
ScopeID uint32
StreamID uint16
Key []byte
}
// DcpScopeDeletion represents a scope Deletion DCP event from the server
type DcpScopeDeletion struct {
SeqNo uint64
Version uint8
VbID uint16
ManifestUID uint64
ScopeID uint32
StreamID uint16
}
// DcpCollectionModification represents a DCP collection modify event from the server
type DcpCollectionModification struct {
SeqNo uint64
ManifestUID uint64
CollectionID uint32
Ttl uint32
VbID uint16
StreamID uint16
Version uint8
}
// DcpOSOSnapshot reprensents a DCP OSSSnapshot from the server
type DcpOSOSnapshot struct {
SnapshotType uint32
VbID uint16
StreamID uint16
}
// DcpSeqNoAdvanced represents a DCP SeqNoAdvanced from the server
type DcpSeqNoAdvanced struct {
SeqNo uint64
VbID uint16
StreamID uint16
}
// DcpStreamEnd represnets a DCP stream end from the server
type DcpStreamEnd struct {
VbID uint16
StreamID uint16
}
// StreamObserver provides an interface to receive events from a running DCP stream.
type StreamObserver interface {
SnapshotMarker(snapshotMarker DcpSnapshotMarker)
Mutation(mutation DcpMutation)
Deletion(deletion DcpDeletion)
Expiration(expiration DcpExpiration)
End(end DcpStreamEnd, err error)
CreateCollection(creation DcpCollectionCreation)
DeleteCollection(deletion DcpCollectionDeletion)
FlushCollection(flush DcpCollectionFlush)
CreateScope(creation DcpScopeCreation)
DeleteScope(deletion DcpScopeDeletion)
ModifyCollection(modification DcpCollectionModification)
OSOSnapshot(snapshot DcpOSOSnapshot)
SeqNoAdvanced(seqNoAdvanced DcpSeqNoAdvanced)
}
type streamFilter struct {
ManifestUID string `json:"uid,omitempty"`
Collections []string `json:"collections,omitempty"`
Scope string `json:"scope,omitempty"`
StreamID uint16 `json:"sid,omitempty"`
}
// OpenStreamCallback is invoked with the results of `OpenStream` operations.
type OpenStreamCallback func([]FailoverEntry, error)
// CloseStreamCallback is invoked with the results of `CloseStream` operations.
type CloseStreamCallback func(error)
// GetFailoverLogCallback is invoked with the results of `GetFailoverLog` operations.
type GetFailoverLogCallback func([]FailoverEntry, error)
// VbSeqNoEntry represents a single GetVbucketSeqnos sequence number entry.
type VbSeqNoEntry struct {
VbID uint16
SeqNo SeqNo
}
// GetVBucketSeqnosCallback is invoked with the results of `GetVBucketSeqnos` operations.
type GetVBucketSeqnosCallback func([]VbSeqNoEntry, error)
gocbcore-10.2.3/dcpagent.go 0000664 0000000 0000000 00000041677 14417540156 0015522 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"fmt"
"sync"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
// DCPAgent represents the base client handling DCP connections to a Couchbase Server.
type DCPAgent struct {
clientID string
bucketName string
pollerController configPollerController
kvMux *kvMux
httpMux *httpMux
dialer *memdClientDialerComponent
cfgManager *configManagementComponent
errMap *errMapComponent
tracer *tracerComponent
diagnostics *diagnosticsComponent
dcp *dcpComponent
http *httpComponent
// These connection settings are only ever changed when ForceReconnect or ReconfigureSecurity are called.
connectionSettingsLock sync.Mutex
auth AuthProvider
authMechanisms []AuthMechanism
tlsConfig *dynTLSConfig
srvDetails *srvDetails
shutdownSig chan struct{}
}
// CreateDcpAgent creates an agent for performing DCP operations.
func CreateDcpAgent(config *DCPAgentConfig, dcpStreamName string, openFlags memd.DcpOpenFlag) (*DCPAgent, error) {
logInfof("SDK Version: gocbcore/%s", goCbCoreVersionStr)
logInfof("Creating new dcp agent: %+v", config)
userAgent := config.UserAgent
disableDecompression := config.CompressionConfig.DisableDecompression
useCompression := config.CompressionConfig.Enabled
useCollections := config.IoConfig.UseCollections
useJSONHello := !config.IoConfig.DisableJSONHello
usePITRHello := config.IoConfig.EnablePITRHello
useXErrorHello := !config.IoConfig.DisableXErrorHello
useSyncReplicationHello := !config.IoConfig.DisableSyncReplicationHello
dcpBufferSize := 20 * 1024 * 1024
compressionMinSize := 32
compressionMinRatio := 0.83
dcpBackfillOrderStr := ""
dcpPriorityStr := ""
kvConnectTimeout := 7000 * time.Millisecond
if config.KVConfig.ConnectTimeout > 0 {
kvConnectTimeout = config.KVConfig.ConnectTimeout
}
serverWaitTimeout := 5 * time.Second
kvPoolSize := 1
if config.KVConfig.PoolSize > 0 {
kvPoolSize = config.KVConfig.PoolSize
}
maxQueueSize := 2048
if config.KVConfig.MaxQueueSize > 0 {
maxQueueSize = config.KVConfig.MaxQueueSize
}
kvBufferSize := uint(0)
if config.KVConfig.ConnectionBufferSize > 0 {
kvBufferSize = config.KVConfig.ConnectionBufferSize
}
confCccpMaxWait := 3 * time.Second
if config.ConfigPollerConfig.CccpMaxWait > 0 {
confCccpMaxWait = config.ConfigPollerConfig.CccpMaxWait
}
confCccpPollPeriod := 2500 * time.Millisecond
if config.ConfigPollerConfig.CccpPollPeriod > 0 {
confCccpPollPeriod = config.ConfigPollerConfig.CccpPollPeriod
}
confHTTPRetryDelay := 10 * time.Second
if config.ConfigPollerConfig.HTTPRetryDelay > 0 {
confHTTPRetryDelay = config.ConfigPollerConfig.HTTPRetryDelay
}
confHTTPRedialPeriod := 10 * time.Second
if config.ConfigPollerConfig.HTTPRedialPeriod > 0 {
confHTTPRedialPeriod = config.ConfigPollerConfig.HTTPRedialPeriod
}
confHTTPMaxWait := 5 * time.Second
if config.ConfigPollerConfig.HTTPMaxWait > 0 {
confHTTPMaxWait = config.ConfigPollerConfig.HTTPMaxWait
}
if config.CompressionConfig.MinSize > 0 {
compressionMinSize = config.CompressionConfig.MinSize
}
if config.CompressionConfig.MinRatio > 0 {
compressionMinRatio = config.CompressionConfig.MinRatio
if compressionMinRatio >= 1.0 {
compressionMinRatio = 1.0
}
}
if config.DCPConfig.BufferSize > 0 {
dcpBufferSize = config.DCPConfig.BufferSize
}
dcpQueueSize := (dcpBufferSize + 23) / 24
switch config.DCPConfig.AgentPriority {
case DcpAgentPriorityLow:
dcpPriorityStr = "low"
case DcpAgentPriorityMed:
dcpPriorityStr = "medium"
case DcpAgentPriorityHigh:
dcpPriorityStr = "high"
}
// If the user doesn't explicitly set the backfill order, the DCP control flag will not be sent to the cluster
// and the default will implicitly be used (which is 'round-robin').
switch config.DCPConfig.BackfillOrder {
case DCPBackfillOrderRoundRobin:
dcpBackfillOrderStr = "round-robin"
case DCPBackfillOrderSequential:
dcpBackfillOrderStr = "sequential"
}
tracerCmpt := newTracerComponent(noopTracer{}, config.BucketName, false, nil)
c := &DCPAgent{
clientID: formatCbUID(randomCbUID()),
bucketName: config.BucketName,
tracer: tracerCmpt,
errMap: newErrMapManager(config.BucketName),
auth: config.SecurityConfig.Auth,
shutdownSig: make(chan struct{}),
}
tlsConfig, err := setupTLSConfig(config.SeedConfig.MemdAddrs, config.SecurityConfig)
if err != nil {
return nil, err
}
c.tlsConfig = tlsConfig
c.authMechanisms = authMechanismsFromConfig(config.SecurityConfig.AuthMechanisms, config.SecurityConfig.UseTLS)
circuitBreakerConfig := CircuitBreakerConfig{
Enabled: false,
}
httpEpList := routeEndpoints{}
var srcHTTPAddrs []routeEndpoint
for _, hostPort := range config.SeedConfig.HTTPAddrs {
if config.SecurityConfig.UseTLS && !config.SecurityConfig.NoTLSSeedNode {
ep := routeEndpoint{
Address: fmt.Sprintf("https://%s", hostPort),
IsSeedNode: true,
}
httpEpList.SSLEndpoints = append(httpEpList.SSLEndpoints, ep)
srcHTTPAddrs = append(srcHTTPAddrs, ep)
} else {
ep := routeEndpoint{
Address: fmt.Sprintf("http://%s", hostPort),
IsSeedNode: true,
}
httpEpList.NonSSLEndpoints = append(httpEpList.NonSSLEndpoints, ep)
srcHTTPAddrs = append(srcHTTPAddrs, ep)
}
}
kvServerList := routeEndpoints{}
var srcMemdAddrs []routeEndpoint
for _, seed := range config.SeedConfig.MemdAddrs {
if config.SecurityConfig.UseTLS && !config.SecurityConfig.NoTLSSeedNode {
kvServerList.SSLEndpoints = append(kvServerList.SSLEndpoints, routeEndpoint{
Address: seed,
IsSeedNode: true,
})
srcMemdAddrs = kvServerList.SSLEndpoints
} else {
kvServerList.NonSSLEndpoints = append(kvServerList.NonSSLEndpoints, routeEndpoint{
Address: seed,
IsSeedNode: true,
})
srcMemdAddrs = kvServerList.NonSSLEndpoints
}
}
if config.SeedConfig.SRVRecord != nil {
c.srvDetails = &srvDetails{
Addrs: kvServerList,
Record: *config.SeedConfig.SRVRecord,
}
}
httpConnectTimeout := 30 * time.Second
if config.HTTPConfig.ConnectTimeout > 0 {
httpConnectTimeout = config.HTTPConfig.ConnectTimeout
}
c.cfgManager = newConfigManager(
configManagerProperties{
NetworkType: config.IoConfig.NetworkType,
SrcMemdAddrs: srcMemdAddrs,
SrcHTTPAddrs: srcHTTPAddrs,
UseTLS: tlsConfig != nil,
NoTLSSeedNode: config.SecurityConfig.NoTLSSeedNode,
},
)
c.dialer = newMemdClientDialerComponent(
memdClientDialerProps{
ServerWaitTimeout: serverWaitTimeout,
KVConnectTimeout: kvConnectTimeout,
ClientID: c.clientID,
DCPQueueSize: dcpQueueSize,
CompressionMinSize: compressionMinSize,
CompressionMinRatio: compressionMinRatio,
DisableDecompression: disableDecompression,
NoTLSSeedNode: config.SecurityConfig.NoTLSSeedNode,
ConnBufSize: kvBufferSize,
DCPBootstrapProps: &memdBootstrapDCPProps{
openFlags: openFlags,
streamName: dcpStreamName,
disableBufferAcknowledgement: config.DCPConfig.DisableBufferAcknowledgement,
useOSOBackfill: config.DCPConfig.UseOSOBackfill,
useStreamID: config.DCPConfig.UseStreamID,
useExpiryOpcode: config.DCPConfig.UseExpiryOpcode,
backfillOrderStr: dcpBackfillOrderStr,
priorityStr: dcpPriorityStr,
bufferSize: dcpBufferSize,
},
},
bootstrapProps{
HelloProps: helloProps{
CollectionsEnabled: useCollections,
CompressionEnabled: useCompression,
JSONFeatureEnabled: useJSONHello,
PITRFeatureEnabled: usePITRHello,
XErrorFeatureEnabled: useXErrorHello,
SyncReplicationEnabled: useSyncReplicationHello,
},
Bucket: c.bucketName,
UserAgent: userAgent,
ErrMapManager: c.errMap,
},
circuitBreakerConfig,
nil,
c.tracer,
c.cfgManager,
)
c.kvMux = newKVMux(
kvMuxProps{
QueueSize: maxQueueSize,
PoolSize: kvPoolSize,
CollectionsEnabled: useCollections,
NoTLSSeedNode: config.SecurityConfig.NoTLSSeedNode,
},
c.cfgManager,
c.errMap,
c.tracer,
c.dialer,
&kvMuxState{
tlsConfig: tlsConfig,
authMechanisms: c.authMechanisms,
auth: config.SecurityConfig.Auth,
},
)
c.httpMux = newHTTPMux(
circuitBreakerConfig,
c.cfgManager,
&httpClientMux{tlsConfig: tlsConfig, auth: config.SecurityConfig.Auth},
config.SecurityConfig.NoTLSSeedNode,
)
c.http = newHTTPComponent(
httpComponentProps{
UserAgent: userAgent,
},
httpClientProps{
maxIdleConns: config.HTTPConfig.MaxIdleConns,
maxIdleConnsPerHost: config.HTTPConfig.MaxIdleConnsPerHost,
idleTimeout: config.HTTPConfig.IdleConnectionTimeout,
connectTimeout: httpConnectTimeout,
},
c.httpMux,
c.tracer,
)
var poller configPollerController
if config.SecurityConfig.NoTLSSeedNode {
poller = newSeedConfigController(srcHTTPAddrs[0].Address, c.bucketName,
httpPollerProperties{
httpComponent: c.http,
confHTTPRetryDelay: confHTTPRetryDelay,
confHTTPRedialPeriod: confHTTPRedialPeriod,
confHTTPMaxWait: confHTTPMaxWait,
}, c.cfgManager)
} else {
var httpPoller *httpConfigController
if c.bucketName != "" {
httpPoller = newHTTPConfigController(
c.bucketName,
httpPollerProperties{
httpComponent: c.http,
confHTTPRetryDelay: confHTTPRetryDelay,
confHTTPRedialPeriod: confHTTPRedialPeriod,
confHTTPMaxWait: confHTTPMaxWait,
},
c.httpMux,
c.cfgManager,
)
}
poller = newPollerController(
newCCCPConfigController(
cccpPollerProperties{
confCccpMaxWait: confCccpMaxWait,
confCccpPollPeriod: confCccpPollPeriod,
},
c.kvMux,
c.cfgManager,
c.isPollingFallbackError,
c.onCCCPNoConfigFromAnyNode,
),
httpPoller,
c.cfgManager,
c.isPollingFallbackError,
)
}
c.pollerController = poller
c.diagnostics = newDiagnosticsComponent(c.kvMux, nil, nil, c.bucketName, newFailFastRetryStrategy(), c.pollerController)
c.dcp = newDcpComponent(c.kvMux, config.DCPConfig.UseStreamID)
c.dialer.AddBootstrapFailHandler(c.diagnostics)
c.dialer.AddCCCPUnsupportedHandler(c)
// Kick everything off.
cfg := &routeConfig{
kvServerList: kvServerList,
mgmtEpList: httpEpList,
revID: -1,
}
c.httpMux.OnNewRouteConfig(cfg)
c.kvMux.OnNewRouteConfig(cfg)
if c.pollerController != nil {
go c.pollerController.Run()
}
return c, nil
}
// IsSecure returns whether this client is connected via SSL.
func (agent *DCPAgent) IsSecure() bool {
return agent.kvMux.IsSecure()
}
// Close shuts down the agent, disconnecting from all servers and failing
// any outstanding operations with ErrShutdown.
func (agent *DCPAgent) Close() error {
routeCloseErr := agent.kvMux.Close()
agent.pollerController.Stop()
// Wait for our external looper goroutines to finish, note that if the
// specific looper wasn't used, it will be a nil value otherwise it
// will be an open channel till its closed to signal completion.
<-agent.pollerController.Done()
return routeCloseErr
}
// WaitUntilReady is used to verify that the SDK has been able to establish connections to the cluster.
// If no strategy is set then a fast fail retry strategy will be applied - only RetryReason that are set to always
// retry will be retried. This is includes for WaitUntilReady, that is the SDK will wait until connections succeed
// or report a connection error - as soon as a connection error is reported WaitUntilReady will fail and return that
// error.
// Connection time errors are also be subject to KvConfig.ServerWaitBackoff. This is the period of time that the SDK
// will wait before attempting to reconnect to a node.
func (agent *DCPAgent) WaitUntilReady(deadline time.Time, opts WaitUntilReadyOptions,
cb WaitUntilReadyCallback) (PendingOp, error) {
forceWait := true
if len(opts.ServiceTypes) == 0 {
forceWait = false
opts.ServiceTypes = []ServiceType{MemdService}
}
return agent.diagnostics.WaitUntilReady(deadline, forceWait, opts, cb)
}
// OpenStream opens a DCP stream for a particular VBucket and, optionally, filter.
func (agent *DCPAgent) OpenStream(vbID uint16, flags memd.DcpStreamAddFlag, vbUUID VbUUID, startSeqNo,
endSeqNo, snapStartSeqNo, snapEndSeqNo SeqNo, evtHandler StreamObserver, opts OpenStreamOptions,
cb OpenStreamCallback) (PendingOp, error) {
return agent.dcp.OpenStream(vbID, flags, vbUUID, startSeqNo, endSeqNo, snapStartSeqNo, snapEndSeqNo, evtHandler, opts, cb)
}
// CloseStream shuts down an open stream for the specified VBucket.
func (agent *DCPAgent) CloseStream(vbID uint16, opts CloseStreamOptions, cb CloseStreamCallback) (PendingOp, error) {
return agent.dcp.CloseStream(vbID, opts, cb)
}
// GetFailoverLog retrieves the fail-over log for a particular VBucket. This is used
// to resume an interrupted stream after a node fail-over has occurred.
func (agent *DCPAgent) GetFailoverLog(vbID uint16, cb GetFailoverLogCallback) (PendingOp, error) {
return agent.dcp.GetFailoverLog(vbID, cb)
}
// GetVbucketSeqnos returns the last checkpoint for a particular VBucket. This is useful
// for starting a DCP stream from wherever the server currently is.
func (agent *DCPAgent) GetVbucketSeqnos(serverIdx int, state memd.VbucketState, opts GetVbucketSeqnoOptions,
cb GetVBucketSeqnosCallback) (PendingOp, error) {
return agent.dcp.GetVbucketSeqnos(serverIdx, state, opts, cb)
}
// HasCollectionsSupport verifies whether or not collections are available on the agent.
func (agent *DCPAgent) HasCollectionsSupport() bool {
return agent.kvMux.SupportsCollections()
}
// ConfigSnapshot returns a snapshot of the underlying configuration currently in use.
func (agent *DCPAgent) ConfigSnapshot() (*ConfigSnapshot, error) {
return agent.kvMux.ConfigSnapshot()
}
// ForceReconnect gracefully rebuilds all connections being used by the agent.
// Any persistent in flight requests (e.g. DCP) will be terminated with ErrForcedReconnect.
//
// Internal: This should never be used and is not supported.
func (agent *DCPAgent) ForceReconnect() {
agent.connectionSettingsLock.Lock()
auth := agent.auth
mechs := agent.authMechanisms
tlsConfig := agent.tlsConfig
agent.connectionSettingsLock.Unlock()
agent.kvMux.ForceReconnect(tlsConfig, mechs, auth, true)
}
// ReconfigureSecurity updates the security configuration being used by the agent. This includes the ability to
// toggle TLS on and off.
//
// Calling this function will cause all underlying connections to be reconnected. The exception to this is the
// connection to the seed node (usually localhost), which will only be reconnected if the AuthProvider is provided
// on the options.
//
// This function can only be called when the seed poller is in use i.e. when the ns_server scheme is used.
// Internal: This should never be used and is not supported.
func (agent *DCPAgent) ReconfigureSecurity(opts ReconfigureSecurityOptions) error {
_, ok := agent.pollerController.(*seedConfigController)
if !ok {
return errors.New("reconfigure tls is only supported when the agent is in ns server mode")
}
var authProvided bool
auth := opts.Auth
mechs := opts.AuthMechanisms
agent.connectionSettingsLock.Lock()
if auth == nil {
auth = agent.auth
} else {
authProvided = true
}
if len(mechs) == 0 {
mechs = agent.authMechanisms
}
var tlsConfig *dynTLSConfig
if opts.UseTLS {
if opts.TLSRootCAProvider == nil {
return wrapError(errInvalidArgument, "must provide TLSRootCAProvider when UseTLS is true")
}
tlsConfig = createTLSConfig(auth, opts.TLSRootCAProvider)
}
agent.auth = auth
agent.authMechanisms = mechs
agent.tlsConfig = tlsConfig
agent.connectionSettingsLock.Unlock()
agent.cfgManager.UseTLS(tlsConfig != nil)
agent.kvMux.ForceReconnect(tlsConfig, mechs, auth, authProvided)
agent.httpMux.UpdateTLS(tlsConfig, auth)
return nil
}
func (agent *DCPAgent) onCCCPUnsupported(err error) {
// If this error is a legitimate fallback reason then we should immediately start the http poller.
if agent.pollerController != nil && agent.isPollingFallbackError(err) {
agent.pollerController.ForceHTTPPoller()
}
}
func (agent *DCPAgent) isPollingFallbackError(err error) bool {
return isPollingFallbackError(err, agent.bucketName)
}
func (agent *DCPAgent) srv() *srvDetails {
return agent.srvDetails
}
func (agent *DCPAgent) setSRVAddrs(addrs routeEndpoints) {
agent.srvDetails.Addrs = addrs
}
func (agent *DCPAgent) routeConfigWatchers() []routeConfigWatcher {
return agent.cfgManager.Watchers()
}
func (agent *DCPAgent) resetConfig() {
// Reset the config manager to accept the next config that the poller fetches.
// This is safe to do here, we're blocking the poller from fetching a config and if we're here then
// we can't be performing ops.
agent.cfgManager.ResetConfig()
// Reset the dialer so that the next connections to bootstrap fetch a config and kick off the poller again.
agent.dialer.ResetConfig()
}
func (agent *DCPAgent) onCCCPNoConfigFromAnyNode(err error) {
onCCCPNoConfigFromAnyNode(agent, err)
}
func (agent *DCPAgent) stopped() <-chan struct{} {
return agent.shutdownSig
}
gocbcore-10.2.3/dcpagent_config.go 0000664 0000000 0000000 00000012205 14417540156 0017030 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"fmt"
"github.com/couchbase/gocbcore/v10/connstr"
"strconv"
)
// DCPAgentConfig specifies the configuration options for creation of a DCPAgent.
type DCPAgentConfig struct {
UserAgent string
BucketName string
SeedConfig SeedConfig
SecurityConfig SecurityConfig
CompressionConfig CompressionConfig
ConfigPollerConfig ConfigPollerConfig
IoConfig IoConfig
KVConfig KVConfig
HTTPConfig HTTPConfig
DCPConfig DCPConfig
}
// DCPConfig specifies DCP specific configuration options.
type DCPConfig struct {
AgentPriority DcpAgentPriority
UseExpiryOpcode bool
UseStreamID bool
UseOSOBackfill bool
BackfillOrder DCPBackfillOrder
BufferSize int
DisableBufferAcknowledgement bool
}
func (config DCPConfig) fromSpec(spec connstr.ResolvedConnSpec) (DCPConfig, error) {
// This option is experimental
if valStr, ok := fetchOption(spec, "dcp_priority"); ok {
var priority DcpAgentPriority
switch valStr {
case "":
priority = DcpAgentPriorityLow
case "low":
priority = DcpAgentPriorityLow
case "medium":
priority = DcpAgentPriorityMed
case "high":
priority = DcpAgentPriorityHigh
default:
return DCPConfig{}, fmt.Errorf("dcp_priority must be one of low, medium or high")
}
config.AgentPriority = priority
}
// This option is experimental
if valStr, ok := fetchOption(spec, "dcp_buffer_size"); ok {
val, err := strconv.ParseInt(valStr, 10, 64)
if err != nil {
return DCPConfig{}, fmt.Errorf("dcp buffer size option must be a number")
}
config.BufferSize = int(val)
}
// This option is experimental
if valStr, ok := fetchOption(spec, "enable_dcp_expiry"); ok {
val, err := strconv.ParseBool(valStr)
if err != nil {
return DCPConfig{}, fmt.Errorf("enable_dcp_expiry option must be a boolean")
}
config.UseExpiryOpcode = val
}
return config, nil
}
func (config *DCPAgentConfig) redacted() interface{} {
newConfig := *config
if isLogRedactionLevelFull() {
newConfig.SeedConfig = newConfig.SeedConfig.redacted()
if newConfig.BucketName != "" {
newConfig.BucketName = redactMetaData(newConfig.BucketName)
}
}
return newConfig
}
// FromConnStr populates the AgentConfig with information from a
// Couchbase Connection String.
// Supported options are:
// ca_cert_path (string) - Specifies the path to a CA certificate.
// network (string) - The network type to use.
// kv_connect_timeout (duration) - Maximum period to attempt to connect to cluster in ms.
// config_poll_interval (duration) - Period to wait between CCCP config polling in ms.
// config_poll_timeout (duration) - Maximum period of time to wait for a CCCP request.
// compression (bool) - Whether to enable network-wise compression of documents.
// compression_min_size (int) - The minimal size of the document in bytes to consider compression.
// compression_min_ratio (float64) - The minimal compress ratio (compressed / original) for the document to be sent compressed.
// orphaned_response_logging (bool) - Whether to enable orphaned response logging.
// orphaned_response_logging_interval (duration) - How often to print the orphan log records.
// orphaned_response_logging_sample_size (int) - The maximum number of orphan log records to track.
// dcp_priority (int) - Specifies the priority to request from the Cluster when connecting for DCP.
// enable_dcp_expiry (bool) - Whether to enable the feature to distinguish between explicit delete and expired delete on DCP.
// kv_pool_size (int) - The number of connections to create to each kv node.
// max_queue_size (int) - The maximum number of requests that can be queued for sending per connection.
// max_idle_http_connections (int) - Maximum number of idle http connections in the pool.
// max_perhost_idle_http_connections (int) - Maximum number of idle http connections in the pool per host.
// idle_http_connection_timeout (duration) - Maximum length of time for an idle connection to stay in the pool in ms.
// http_redial_period (duration) - The maximum length of time for the HTTP poller to stay connected before reconnecting.
// http_retry_delay (duration) - The length of time to wait between HTTP poller retries if connecting fails.
func (config *DCPAgentConfig) FromConnStr(connStr string) error {
baseSpec, err := connstr.Parse(connStr)
if err != nil {
return err
}
spec, err := connstr.Resolve(baseSpec)
if err != nil {
return err
}
config.DCPConfig, err = config.DCPConfig.fromSpec(spec)
if err != nil {
return err
}
config.SeedConfig, err = config.SeedConfig.fromSpec(spec)
if err != nil {
return err
}
config.SecurityConfig, err = config.SecurityConfig.fromSpec(spec)
if err != nil {
return err
}
config.CompressionConfig, err = config.CompressionConfig.fromSpec(spec)
if err != nil {
return err
}
config.ConfigPollerConfig, err = config.ConfigPollerConfig.fromSpec(spec)
if err != nil {
return err
}
config.IoConfig, err = config.IoConfig.fromSpec(spec)
if err != nil {
return err
}
config.HTTPConfig, err = config.HTTPConfig.fromSpec(spec)
if err != nil {
return err
}
config.KVConfig, err = config.KVConfig.fromSpec(spec)
if err != nil {
return err
}
return nil
}
gocbcore-10.2.3/dcpagent_config_test.go 0000664 0000000 0000000 00000032310 14417540156 0020066 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"testing"
"time"
)
func (suite *StandardTestSuite) TestDCPAgentConfig_FromConnStr() {
connStr := "couchbase://10.112.192.101,10.112.192.102?bootstrap_on=cccp&network=external&kv_connect_timeout=100us"
config := &DCPAgentConfig{}
err := config.FromConnStr(connStr)
if err != nil {
suite.T().Fatalf("Failed to execute FromConnStr: %v", err)
}
if config.KVConfig.ConnectTimeout != 100*time.Microsecond {
suite.T().Fatalf("Ex :%v", config.KVConfig.ConnectTimeout)
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_Couchbase1() {
connStr := "couchbase://10.112.192.101"
config := &DCPAgentConfig{}
err := config.FromConnStr(connStr)
if err != nil {
suite.T().Fatalf("Failed to execute FromConnStr: %v", err)
}
if len(config.SeedConfig.MemdAddrs) != 1 {
suite.T().Fatalf("Expected MemdAddrs to be len 1 but was %v", config.SeedConfig.MemdAddrs)
}
if config.SeedConfig.MemdAddrs[0] != "10.112.192.101:11210" {
suite.T().Fatalf("Expected address to be 10.112.192.101:11210 but was %v", config.SeedConfig.MemdAddrs[0])
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_Couchbase2() {
connStr := "couchbase://10.112.192.101,10.112.192.102"
config := &DCPAgentConfig{}
err := config.FromConnStr(connStr)
if err != nil {
suite.T().Fatalf("Failed to execute FromConnStr: %v", err)
}
if len(config.SeedConfig.MemdAddrs) != 2 {
suite.T().Fatalf("Expected MemdAddrs to be len 2 but was %v", config.SeedConfig.MemdAddrs)
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_DefaultHTTP() {
connStr := "http://10.112.192.101:8091"
config := &DCPAgentConfig{}
err := config.FromConnStr(connStr)
if err != nil {
suite.T().Fatalf("Failed to execute FromConnStr: %v", err)
}
if len(config.SeedConfig.MemdAddrs) != 1 {
suite.T().Fatalf("Expected MemdAddrs to be len 1 but was %v", config.SeedConfig.MemdAddrs)
}
if len(config.SeedConfig.HTTPAddrs) != 1 {
suite.T().Fatalf("Expected MemdAddrs to be len 1 but was %v", config.SeedConfig.HTTPAddrs)
}
if config.SeedConfig.MemdAddrs[0] != "10.112.192.101:11210" {
suite.T().Fatalf("Expected address to be 10.112.192.101:11210 but was %v", config.SeedConfig.MemdAddrs[0])
}
if config.SeedConfig.HTTPAddrs[0] != "10.112.192.101:8091" {
suite.T().Fatalf("Expected address to be 10.112.192.101:8091 but was %v", config.SeedConfig.HTTPAddrs[0])
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_NonDefaultHTTP() {
connStr := "http://10.112.192.101:9000"
config := &DCPAgentConfig{}
err := config.FromConnStr(connStr)
if err != nil {
suite.T().Fatalf("Failed to execute FromConnStr: %v", err)
}
if len(config.SeedConfig.MemdAddrs) != 0 {
suite.T().Fatalf("Expected MemdAddrs to be len 0 but was %v", config.SeedConfig.MemdAddrs)
}
if len(config.SeedConfig.HTTPAddrs) != 1 {
suite.T().Fatalf("Expected MemdAddrs to be len 1 but was %v", config.SeedConfig.HTTPAddrs)
}
if config.SeedConfig.HTTPAddrs[0] != "10.112.192.101:9000" {
suite.T().Fatalf("Expected address to be 10.112.192.101:9000 but was %v", config.SeedConfig.HTTPAddrs[0])
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_Network() {
tests := []struct {
name string
connStr string
expected string
}{
{
name: "external",
connStr: "couchbase://10.112.192.101?network=external",
expected: "external",
},
{
name: "default",
connStr: "couchbase://10.112.192.101?network=default",
expected: "default",
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); err != nil {
t.Errorf("FromConnStr() error = %v", err)
}
if config.IoConfig.NetworkType != tt.expected {
suite.T().Fatalf("Expected %s but was %s", tt.expected, config.IoConfig.NetworkType)
}
})
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_KVConnectTimeout() {
tests := []struct {
name string
connStr string
expected time.Duration
wantErr bool
}{
{
name: "duration",
connStr: "couchbase://10.112.192.101?kv_connect_timeout=5000us",
expected: 5 * time.Millisecond,
},
{
name: "ms",
connStr: "couchbase://10.112.192.101?kv_connect_timeout=5",
expected: 5 * time.Millisecond,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?kv_connect_timeout=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if config.KVConfig.ConnectTimeout != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.KVConfig.ConnectTimeout)
}
})
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_ConfigPollTimeout() {
tests := []struct {
name string
connStr string
expected time.Duration
wantErr bool
}{
{
name: "duration",
connStr: "couchbase://10.112.192.101?config_poll_timeout=5000us",
expected: 5 * time.Millisecond,
},
{
name: "ms",
connStr: "couchbase://10.112.192.101?config_poll_timeout=5",
expected: 5 * time.Millisecond,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?config_poll_timeout=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if config.ConfigPollerConfig.CccpMaxWait != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.ConfigPollerConfig.CccpMaxWait)
}
})
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_ConfigPollPeriod() {
tests := []struct {
name string
connStr string
expected time.Duration
wantErr bool
}{
{
name: "duration",
connStr: "couchbase://10.112.192.101?config_poll_interval=5000us",
expected: 5 * time.Millisecond,
},
{
name: "ms",
connStr: "couchbase://10.112.192.101?config_poll_interval=5",
expected: 5 * time.Millisecond,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?config_poll_interval=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.ConfigPollerConfig.CccpPollPeriod != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.ConfigPollerConfig.CccpPollPeriod)
}
})
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_Compression() {
tests := []struct {
name string
connStr string
expected bool
wantErr bool
}{
{
name: "true",
connStr: "couchbase://10.112.192.101?compression=true",
expected: true,
},
{
name: "false",
connStr: "couchbase://10.112.192.101?compression=false",
expected: false,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?compression=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.CompressionConfig.Enabled != tt.expected {
suite.T().Fatalf("Expected %t but was %t", tt.expected, config.CompressionConfig.Enabled)
}
})
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_CompressionMinSize() {
tests := []struct {
name string
connStr string
expected int
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?compression_min_size=100000",
expected: 100000,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?compression_min_size=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.CompressionConfig.MinSize != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.CompressionConfig.MinSize)
}
})
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_CompressionMinRatio() {
tests := []struct {
name string
connStr string
expected float64
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?compression_min_ratio=0.7",
expected: 0.7,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?compression_min_ratio=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.CompressionConfig.MinRatio != tt.expected {
suite.T().Fatalf("Expected %f but was %f", tt.expected, config.CompressionConfig.MinRatio)
}
})
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_DCPPriority() {
tests := []struct {
name string
connStr string
expected DcpAgentPriority
wantErr bool
}{
{
name: "empty",
connStr: "couchbase://10.112.192.101?dcp_priority=",
expected: DcpAgentPriorityLow,
},
{
name: "low",
connStr: "couchbase://10.112.192.101?dcp_priority=low",
expected: DcpAgentPriorityLow,
},
{
name: "medium",
connStr: "couchbase://10.112.192.101?dcp_priority=medium",
expected: DcpAgentPriorityMed,
},
{
name: "high",
connStr: "couchbase://10.112.192.101?dcp_priority=high",
expected: DcpAgentPriorityHigh,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?dcp_priority=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.DCPConfig.AgentPriority != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.DCPConfig.AgentPriority)
}
})
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_EnableDCPExpiry() {
tests := []struct {
name string
connStr string
expected bool
wantErr bool
}{
{
name: "true",
connStr: "couchbase://10.112.192.101?enable_dcp_expiry=true",
expected: true,
},
{
name: "false",
connStr: "couchbase://10.112.192.101?enable_dcp_expiry=false",
expected: false,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?enable_dcp_expiry=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.DCPConfig.UseExpiryOpcode != tt.expected {
suite.T().Fatalf("Expected %t but was %t", tt.expected, config.DCPConfig.UseExpiryOpcode)
}
})
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_KVPoolSize() {
tests := []struct {
name string
connStr string
expected int
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?kv_pool_size=2",
expected: 2,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?kv_pool_size=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.KVConfig.PoolSize != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.KVConfig.PoolSize)
}
})
}
}
func (suite *StandardTestSuite) TestDCPAgentConfig_MaxQueueSize() {
tests := []struct {
name string
connStr string
expected int
wantErr bool
}{
{
name: "valid",
connStr: "couchbase://10.112.192.101?max_queue_size=2",
expected: 2,
},
{
name: "invalid",
connStr: "couchbase://10.112.192.101?max_queue_size=squirrel",
wantErr: true,
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
config := &DCPAgentConfig{}
if err := config.FromConnStr(tt.connStr); (err != nil) != tt.wantErr {
t.Errorf("FromConnStr() error = %v, wanted error = %t", err, tt.wantErr)
}
if tt.wantErr {
return
}
if config.KVConfig.MaxQueueSize != tt.expected {
suite.T().Fatalf("Expected %d but was %d", tt.expected, config.KVConfig.MaxQueueSize)
}
})
}
}
gocbcore-10.2.3/dcpcomponent.go 0000664 0000000 0000000 00000033514 14417540156 0016415 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"sync/atomic"
"github.com/couchbase/gocbcore/v10/memd"
)
type dcpComponent struct {
kvMux *kvMux
streamIDEnabled bool
}
func newDcpComponent(kvMux *kvMux, streamIDEnabled bool) *dcpComponent {
return &dcpComponent{
kvMux: kvMux,
streamIDEnabled: streamIDEnabled,
}
}
func (dcp *dcpComponent) OpenStream(vbID uint16, flags memd.DcpStreamAddFlag, vbUUID VbUUID, startSeqNo,
endSeqNo, snapStartSeqNo, snapEndSeqNo SeqNo, evtHandler StreamObserver, opts OpenStreamOptions,
cb OpenStreamCallback) (PendingOp, error) {
var req *memdQRequest
var openHandled uint32
handler := func(resp *memdQResponse, _ *memdQRequest, err error) {
if resp == nil && err == nil {
logWarnf("DCP event occurred with no error and no response")
return
}
if err != nil {
if resp == nil {
if atomic.CompareAndSwapUint32(&openHandled, 0, 1) {
// If open hasn't been handled and there's no response then it's reasonably safe to assume that
// this occurring for the open stream request.
cb(nil, err)
return
}
}
if resp != nil && resp.Magic == memd.CmdMagicRes {
// CmdMagicRes means that this must be the open stream request response.
atomic.StoreUint32(&openHandled, 1)
// We need to decorate rollback errors with extra information that the server returns to us.
// Unforunately we have to check for the memd due to earlier oversights where we missed converting
// it to a proper gocbcore error.
if errors.Is(err, ErrMemdRollback) {
err = DCPRollbackError{
InnerError: err,
SeqNo: SeqNo(binary.BigEndian.Uint64(resp.Value)),
}
}
cb(nil, err)
return
}
var streamID uint16
if opts.StreamOptions != nil {
streamID = opts.StreamOptions.StreamID
}
evtHandler.End(DcpStreamEnd{vbID, streamID}, err)
return
}
if resp.Magic == memd.CmdMagicRes {
atomic.StoreUint32(&openHandled, 1)
// This is the response to the open stream request.
numEntries := len(resp.Value) / 16
entries := make([]FailoverEntry, numEntries)
for i := 0; i < numEntries; i++ {
entries[i] = FailoverEntry{
VbUUID: VbUUID(binary.BigEndian.Uint64(resp.Value[i*16+0:])),
SeqNo: SeqNo(binary.BigEndian.Uint64(resp.Value[i*16+8:])),
}
}
cb(entries, nil)
return
}
// This is one of the stream events
switch resp.Command {
case memd.CmdDcpSnapshotMarker:
snapShotmarker := DcpSnapshotMarker{VbID: resp.Vbucket}
if resp.StreamIDFrame != nil {
snapShotmarker.StreamID = resp.StreamIDFrame.StreamID
}
if len(resp.Extras) == 20 {
// Length of 20 indicates a v1 packet
snapShotmarker.StartSeqNo = binary.BigEndian.Uint64(resp.Extras[0:])
snapShotmarker.EndSeqNo = binary.BigEndian.Uint64(resp.Extras[8:])
snapShotmarker.SnapshotType = SnapshotState(binary.BigEndian.Uint32(resp.Extras[16:]))
} else if len(resp.Extras) == 1 {
// Length of 1 indicates a v2 packet
snapShotmarker.StartSeqNo = binary.BigEndian.Uint64(resp.Value[0:])
snapShotmarker.EndSeqNo = binary.BigEndian.Uint64(resp.Value[8:])
snapShotmarker.SnapshotType = SnapshotState(binary.BigEndian.Uint32(resp.Value[16:]))
snapShotmarker.MaxVisibleSeqNo = binary.BigEndian.Uint64(resp.Value[20:])
snapShotmarker.HighCompletedSeqNo = binary.BigEndian.Uint64(resp.Value[28:])
version := int(resp.Extras[0])
if version == 1 {
// v2.1 includes the snapshot TimeStamp
snapShotmarker.SnapshotTimeStamp = binary.BigEndian.Uint64(resp.Value[36:])
}
}
evtHandler.SnapshotMarker(snapShotmarker)
case memd.CmdDcpMutation:
mutation := DcpMutation{
SeqNo: binary.BigEndian.Uint64(resp.Extras[0:]),
RevNo: binary.BigEndian.Uint64(resp.Extras[8:]),
Flags: binary.BigEndian.Uint32(resp.Extras[16:]),
Expiry: binary.BigEndian.Uint32(resp.Extras[20:]),
LockTime: binary.BigEndian.Uint32(resp.Extras[24:]),
Cas: resp.Cas,
Datatype: resp.Datatype,
VbID: resp.Vbucket,
CollectionID: resp.CollectionID,
Key: resp.Key,
Value: resp.Value,
}
if resp.StreamIDFrame != nil {
mutation.StreamID = resp.StreamIDFrame.StreamID
}
evtHandler.Mutation(mutation)
case memd.CmdDcpDeletion:
deletion := DcpDeletion{
SeqNo: binary.BigEndian.Uint64(resp.Extras[0:]),
RevNo: binary.BigEndian.Uint64(resp.Extras[8:]),
Cas: resp.Cas,
Datatype: resp.Datatype,
VbID: resp.Vbucket,
CollectionID: resp.CollectionID,
Key: resp.Key,
Value: resp.Value,
}
if len(resp.Extras) == 21 {
// Length of 21 indicates a v2 packet
deletion.DeleteTime = binary.BigEndian.Uint32(resp.Extras[16:])
}
if resp.StreamIDFrame != nil {
deletion.StreamID = resp.StreamIDFrame.StreamID
}
evtHandler.Deletion(deletion)
case memd.CmdDcpExpiration:
expiration := DcpExpiration{
SeqNo: binary.BigEndian.Uint64(resp.Extras[0:]),
RevNo: binary.BigEndian.Uint64(resp.Extras[8:]),
Cas: resp.Cas,
VbID: resp.Vbucket,
CollectionID: resp.CollectionID,
Key: resp.Key,
}
if len(resp.Extras) > 16 {
expiration.DeleteTime = binary.BigEndian.Uint32(resp.Extras[16:])
}
if resp.StreamIDFrame != nil {
expiration.StreamID = resp.StreamIDFrame.StreamID
}
evtHandler.Expiration(expiration)
case memd.CmdDcpEvent:
vbID := resp.Vbucket
seqNo := binary.BigEndian.Uint64(resp.Extras[0:])
eventCode := memd.StreamEventCode(binary.BigEndian.Uint32(resp.Extras[8:]))
version := resp.Extras[12]
var streamID uint16
if resp.StreamIDFrame != nil {
streamID = resp.StreamIDFrame.StreamID
}
switch eventCode {
case memd.StreamEventCollectionCreate:
creation := DcpCollectionCreation{
SeqNo: seqNo,
Version: version,
VbID: vbID,
ManifestUID: binary.BigEndian.Uint64(resp.Value[0:]),
ScopeID: binary.BigEndian.Uint32(resp.Value[8:]),
CollectionID: binary.BigEndian.Uint32(resp.Value[12:]),
StreamID: streamID,
Key: resp.Key,
}
if version == 1 {
creation.Ttl = binary.BigEndian.Uint32(resp.Value[16:])
}
evtHandler.CreateCollection(creation)
case memd.StreamEventCollectionDelete:
deleteion := DcpCollectionDeletion{
SeqNo: seqNo,
Version: version,
VbID: vbID,
ManifestUID: binary.BigEndian.Uint64(resp.Value[0:]),
ScopeID: binary.BigEndian.Uint32(resp.Value[8:]),
CollectionID: binary.BigEndian.Uint32(resp.Value[12:]),
StreamID: streamID,
}
evtHandler.DeleteCollection(deleteion)
case memd.StreamEventCollectionFlush:
flush := DcpCollectionFlush{
SeqNo: seqNo,
Version: version,
VbID: vbID,
ManifestUID: binary.BigEndian.Uint64(resp.Value[0:]),
CollectionID: binary.BigEndian.Uint32(resp.Value[8:]),
StreamID: streamID,
}
evtHandler.FlushCollection(flush)
case memd.StreamEventScopeCreate:
creation := DcpScopeCreation{
SeqNo: seqNo,
Version: version,
VbID: vbID,
ManifestUID: binary.BigEndian.Uint64(resp.Value[0:]),
ScopeID: binary.BigEndian.Uint32(resp.Value[8:]),
StreamID: streamID,
Key: resp.Key,
}
evtHandler.CreateScope(creation)
case memd.StreamEventScopeDelete:
deletion := DcpScopeDeletion{
SeqNo: seqNo,
Version: version,
VbID: vbID,
ManifestUID: binary.BigEndian.Uint64(resp.Value[0:]),
ScopeID: binary.BigEndian.Uint32(resp.Value[8:]),
StreamID: streamID,
}
evtHandler.DeleteScope(deletion)
case memd.StreamEventCollectionChanged:
modification := DcpCollectionModification{
SeqNo: seqNo,
Version: version,
VbID: vbID,
ManifestUID: binary.BigEndian.Uint64(resp.Value[0:]),
CollectionID: binary.BigEndian.Uint32(resp.Value[8:]),
Ttl: binary.BigEndian.Uint32(resp.Value[12:]),
StreamID: streamID,
}
evtHandler.ModifyCollection(modification)
}
case memd.CmdDcpStreamEnd:
code := memd.StreamEndStatus(binary.BigEndian.Uint32(resp.Extras[0:]))
end := DcpStreamEnd{
VbID: resp.Vbucket,
}
if resp.StreamIDFrame != nil {
end.StreamID = resp.StreamIDFrame.StreamID
}
if req.internalCancel(err) {
evtHandler.End(end, getStreamEndStatusError(code))
}
case memd.CmdDcpOsoSnapshot:
snapshot := DcpOSOSnapshot{
VbID: resp.Vbucket,
SnapshotType: binary.BigEndian.Uint32(resp.Extras[0:]),
}
if resp.StreamIDFrame != nil {
snapshot.StreamID = resp.StreamIDFrame.StreamID
}
evtHandler.OSOSnapshot(snapshot)
case memd.CmdDcpSeqNoAdvanced:
seqNoAdvanced := DcpSeqNoAdvanced{
SeqNo: binary.BigEndian.Uint64(resp.Extras[0:]),
VbID: resp.Vbucket,
}
if resp.StreamIDFrame != nil {
seqNoAdvanced.StreamID = resp.StreamIDFrame.StreamID
}
evtHandler.SeqNoAdvanced(seqNoAdvanced)
}
}
extraBuf := make([]byte, 48)
binary.BigEndian.PutUint32(extraBuf[0:], uint32(flags))
binary.BigEndian.PutUint32(extraBuf[4:], 0)
binary.BigEndian.PutUint64(extraBuf[8:], uint64(startSeqNo))
binary.BigEndian.PutUint64(extraBuf[16:], uint64(endSeqNo))
binary.BigEndian.PutUint64(extraBuf[24:], uint64(vbUUID))
binary.BigEndian.PutUint64(extraBuf[32:], uint64(snapStartSeqNo))
binary.BigEndian.PutUint64(extraBuf[40:], uint64(snapEndSeqNo))
var val []byte
val = nil
if opts.StreamOptions != nil || opts.FilterOptions != nil || opts.ManifestOptions != nil {
convertedFilter := streamFilter{}
if opts.FilterOptions != nil {
// If there are collection IDs then we can assume that scope ID of 0 actually means no scope ID
if len(opts.FilterOptions.CollectionIDs) > 0 {
for _, cid := range opts.FilterOptions.CollectionIDs {
convertedFilter.Collections = append(convertedFilter.Collections, fmt.Sprintf("%x", cid))
}
} else {
// No collection IDs but the filter was set so even if scope ID is 0 then we use it
convertedFilter.Scope = fmt.Sprintf("%x", opts.FilterOptions.ScopeID)
}
}
if opts.ManifestOptions != nil {
convertedFilter.ManifestUID = fmt.Sprintf("%x", opts.ManifestOptions.ManifestUID)
}
if opts.StreamOptions != nil {
convertedFilter.StreamID = opts.StreamOptions.StreamID
}
var err error
val, err = json.Marshal(convertedFilter)
if err != nil {
return nil, err
}
}
req = &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdDcpStreamReq,
Datatype: 0,
Cas: 0,
Extras: extraBuf,
Key: nil,
Value: val,
Vbucket: vbID,
},
Callback: handler,
ReplicaIdx: 0,
Persistent: true,
}
return dcp.kvMux.DispatchDirect(req)
}
func (dcp *dcpComponent) CloseStream(vbID uint16, opts CloseStreamOptions, cb CloseStreamCallback) (PendingOp, error) {
handler := func(_ *memdQResponse, _ *memdQRequest, err error) {
cb(err)
}
var streamFrame *memd.StreamIDFrame
if opts.StreamOptions != nil {
if !dcp.streamIDEnabled {
return nil, errStreamIDNotEnabled
}
streamFrame = &memd.StreamIDFrame{
StreamID: opts.StreamOptions.StreamID,
}
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdDcpCloseStream,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: nil,
Value: nil,
Vbucket: vbID,
StreamIDFrame: streamFrame,
},
Callback: handler,
ReplicaIdx: 0,
Persistent: false,
RetryStrategy: newFailFastRetryStrategy(),
}
return dcp.kvMux.DispatchDirect(req)
}
func (dcp *dcpComponent) GetFailoverLog(vbID uint16, cb GetFailoverLogCallback) (PendingOp, error) {
handler := func(resp *memdQResponse, _ *memdQRequest, err error) {
if err != nil {
cb(nil, err)
return
}
numEntries := len(resp.Value) / 16
entries := make([]FailoverEntry, numEntries)
for i := 0; i < numEntries; i++ {
entries[i] = FailoverEntry{
VbUUID: VbUUID(binary.BigEndian.Uint64(resp.Value[i*16+0:])),
SeqNo: SeqNo(binary.BigEndian.Uint64(resp.Value[i*16+8:])),
}
}
cb(entries, nil)
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdDcpGetFailoverLog,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: nil,
Value: nil,
Vbucket: vbID,
},
Callback: handler,
ReplicaIdx: 0,
Persistent: false,
RetryStrategy: newFailFastRetryStrategy(),
}
return dcp.kvMux.DispatchDirect(req)
}
func (dcp *dcpComponent) GetVbucketSeqnos(serverIdx int, state memd.VbucketState, opts GetVbucketSeqnoOptions, cb GetVBucketSeqnosCallback) (PendingOp, error) {
handler := func(resp *memdQResponse, _ *memdQRequest, err error) {
if err != nil {
cb(nil, err)
return
}
var vbs []VbSeqNoEntry
numVbs := len(resp.Value) / 10
for i := 0; i < numVbs; i++ {
vbs = append(vbs, VbSeqNoEntry{
VbID: binary.BigEndian.Uint16(resp.Value[i*10:]),
SeqNo: SeqNo(binary.BigEndian.Uint64(resp.Value[i*10+2:])),
})
}
cb(vbs, nil)
}
var extraBuf []byte
if opts.FilterOptions == nil {
extraBuf = make([]byte, 4)
binary.BigEndian.PutUint32(extraBuf[0:], uint32(state))
} else {
if !dcp.kvMux.SupportsCollections() {
return nil, errCollectionsUnsupported
}
extraBuf = make([]byte, 8)
binary.BigEndian.PutUint32(extraBuf[0:], uint32(state))
binary.BigEndian.PutUint32(extraBuf[4:], opts.FilterOptions.CollectionID)
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGetAllVBSeqnos,
Datatype: 0,
Cas: 0,
Extras: extraBuf,
Key: nil,
Value: nil,
Vbucket: 0,
},
Callback: handler,
ReplicaIdx: -serverIdx,
Persistent: false,
RetryStrategy: newFailFastRetryStrategy(),
}
return dcp.kvMux.DispatchDirect(req)
}
gocbcore-10.2.3/diagnostics.go 0000664 0000000 0000000 00000013200 14417540156 0016221 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"context"
"sort"
"sync"
"sync/atomic"
"time"
)
// PingState is the current state of a endpoint used in a PingResult.
type PingState uint32
const (
// PingStateOK indicates that an endpoint is OK.
PingStateOK PingState = 1
// PingStateTimeout indicates that the ping request to an endpoint timed out.
PingStateTimeout PingState = 2
// PingStateError indicates that the ping request to an endpoint encountered an error.
PingStateError PingState = 3
)
// EndpointState is the current connection state of an endpoint.
type EndpointState uint32
const (
// EndpointStateDisconnected indicates that the endpoint is disconnected.
EndpointStateDisconnected EndpointState = 1
// EndpointStateConnecting indicates that the endpoint is connecting.
EndpointStateConnecting EndpointState = 2
// EndpointStateConnected indicates that the endpoint is connected.
EndpointStateConnected EndpointState = 3
// EndpointStateDisconnecting indicates that the endpoint is disconnecting.
EndpointStateDisconnecting EndpointState = 4
)
// EndpointPingResult contains the results of a ping to a single server.
type EndpointPingResult struct {
Endpoint string
Error error
Latency time.Duration
ID string
Scope string
State PingState
}
type pingSubOp struct {
op PendingOp
endpoint string
}
type pingOp struct {
lock sync.Mutex
subops []pingSubOp
remaining int32
results map[ServiceType][]EndpointPingResult
callback PingCallback
bucketName string
httpCancel context.CancelFunc
}
func (pop *pingOp) Cancel() {
for _, subop := range pop.subops {
subop.op.Cancel()
}
pop.httpCancel()
}
func (pop *pingOp) handledOneLocked(configRev int64) {
remaining := atomic.AddInt32(&pop.remaining, -1)
if remaining == 0 {
pop.httpCancel()
pop.callback(&PingResult{
ConfigRev: configRev,
Services: pop.results,
}, nil)
}
}
// PingOptions encapsulates the parameters for a PingKv operation.
type PingOptions struct {
TraceContext RequestSpanContext
KVDeadline time.Time
CbasDeadline time.Time
N1QLDeadline time.Time
FtsDeadline time.Time
CapiDeadline time.Time
MgmtDeadline time.Time
ServiceTypes []ServiceType
// Internal: This should never be used and is not supported.
User string
ignoreMissingServices bool
}
// PingResult encapsulates the result of a PingKv operation.
type PingResult struct {
ConfigRev int64
Services map[ServiceType][]EndpointPingResult
}
// DiagnosticsOptions encapsulates the parameters for a Diagnostics operation.
type DiagnosticsOptions struct {
}
// MemdConnInfo represents information we know about a particular
// memcached connection reported in a diagnostics report.
type MemdConnInfo struct {
LocalAddr string
RemoteAddr string
LastActivity time.Time
Scope string
ID string
State EndpointState
}
// DiagnosticInfo is returned by the Diagnostics method and includes
// information about the overall health of the clients connections.
type DiagnosticInfo struct {
ConfigRev int64
MemdConns []MemdConnInfo
State ClusterState
}
// ClusterState is used to describe the state of a cluster.
type ClusterState uint32
const (
// ClusterStateOnline specifies that all nodes and their sockets are reachable.
ClusterStateOnline = ClusterState(1)
// ClusterStateDegraded specifies that at least one socket per service is reachable.
ClusterStateDegraded = ClusterState(2)
// ClusterStateOffline is used to specify that not even one socker per service is reachable.
ClusterStateOffline = ClusterState(3)
)
type waitUntilOp struct {
lock sync.Mutex
remaining int32
callback WaitUntilReadyCallback
stopCh chan struct{}
timer *time.Timer
httpCancel context.CancelFunc
closed bool
retryLock sync.Mutex
retries uint32
retryReasons []RetryReason
retryStrat RetryStrategy
}
func (wuo *waitUntilOp) RetryAttempts() uint32 {
return atomic.LoadUint32(&wuo.retries)
}
func (wuo *waitUntilOp) RetryReasons() []RetryReason {
wuo.retryLock.Lock()
defer wuo.retryLock.Unlock()
return wuo.retryReasons
}
func (wuo *waitUntilOp) Identifier() string {
return "waituntilready"
}
func (wuo *waitUntilOp) Idempotent() bool {
return true
}
func (wuo *waitUntilOp) retryStrategy() RetryStrategy {
return wuo.retryStrat
}
func (wuo *waitUntilOp) recordRetryAttempt(reason RetryReason) {
atomic.AddUint32(&wuo.retries, 1)
wuo.retryLock.Lock()
defer wuo.retryLock.Unlock()
idx := sort.Search(len(wuo.retryReasons), func(i int) bool {
return wuo.retryReasons[i] == reason
})
// if idx is out of the range of retryReasons then it wasn't found.
if idx > len(wuo.retryReasons)-1 {
wuo.retryReasons = append(wuo.retryReasons, reason)
}
}
func (wuo *waitUntilOp) cancel(err error) {
wuo.lock.Lock()
wuo.timer.Stop()
if wuo.closed {
wuo.lock.Unlock()
return
}
wuo.closed = true
wuo.lock.Unlock()
close(wuo.stopCh)
wuo.httpCancel()
wuo.callback(nil, err)
}
func (wuo *waitUntilOp) Cancel() {
wuo.cancel(errRequestCanceled)
}
func (wuo *waitUntilOp) handledOneLocked() {
remaining := atomic.AddInt32(&wuo.remaining, -1)
if remaining == 0 {
wuo.timer.Stop()
wuo.httpCancel()
wuo.callback(&WaitUntilReadyResult{}, nil)
}
}
// WaitUntilReadyResult encapsulates the result of a WaitUntilReady operation.
type WaitUntilReadyResult struct {
}
// WaitUntilReadyOptions encapsulates the parameters for a WaitUntilReady operation.
type WaitUntilReadyOptions struct {
DesiredState ClusterState // Defaults to ClusterStateOnline
ServiceTypes []ServiceType // Defaults to all services
// If the cluster state is offline and a connect error has been observed then fast fail and return it.
RetryStrategy RetryStrategy
}
gocbcore-10.2.3/diagnosticscomponent.go 0000664 0000000 0000000 00000054332 14417540156 0020157 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"context"
"errors"
"fmt"
"io/ioutil"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/couchbase/gocbcore/v10/memd"
)
type diagnosticsComponent struct {
kvMux *kvMux
httpMux *httpMux
httpComponent *httpComponent
bucket string
defaultRetry RetryStrategy
pollerErrorProvider pollerErrorProvider
// preConfigBootstrapError must only be used for checking for bootstrap errors when a config has not yet been seen.
preConfigBootstrapError error
preConfigBootstrapErrorLock sync.Mutex
}
func newDiagnosticsComponent(kvMux *kvMux, httpMux *httpMux, httpComponent *httpComponent, bucket string,
defaultRetry RetryStrategy, pollerErrorProvider pollerErrorProvider) *diagnosticsComponent {
return &diagnosticsComponent{
kvMux: kvMux,
httpMux: httpMux,
bucket: bucket,
httpComponent: httpComponent,
defaultRetry: defaultRetry,
pollerErrorProvider: pollerErrorProvider,
}
}
func (dc *diagnosticsComponent) onBootstrapFail(err error) {
// It doesn't really matter if we overwrite this error.
dc.preConfigBootstrapErrorLock.Lock()
dc.preConfigBootstrapError = err
dc.preConfigBootstrapErrorLock.Unlock()
}
func (dc *diagnosticsComponent) pingKV(ctx context.Context, interval time.Duration, deadline time.Time,
retryStrat RetryStrategy, user string, op *pingOp) {
var userFrame *memd.UserImpersonationFrame
if len(user) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(user),
}
}
if !deadline.IsZero() {
// We have to setup a new child context with its own deadline because services have their own timeout values.
var cancel context.CancelFunc
ctx, cancel = context.WithDeadline(ctx, deadline)
defer cancel()
}
for {
iter, err := dc.kvMux.PipelineSnapshot()
if err != nil {
logErrorf("failed to get pipeline snapshot")
select {
case <-ctx.Done():
ctxErr := ctx.Err()
var cancelReason error
if errors.Is(ctxErr, context.Canceled) {
cancelReason = ctxErr
} else {
cancelReason = errUnambiguousTimeout
}
op.results[MemdService] = append(op.results[MemdService], EndpointPingResult{
Error: cancelReason,
Scope: op.bucketName,
ID: uuid.New().String(),
State: PingStateTimeout,
})
op.handledOneLocked(iter.RevID())
return
case <-time.After(interval):
continue
}
}
if iter.RevID() > -1 {
var wg sync.WaitGroup
iter.Iterate(0, func(p *memdPipeline) bool {
wg.Add(1)
go func(pipeline *memdPipeline) {
serverAddress := pipeline.Address()
startTime := time.Now()
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
pingLatency := time.Since(startTime)
state := PingStateOK
if err != nil {
if errors.Is(err, ErrTimeout) {
state = PingStateTimeout
} else {
state = PingStateError
}
}
op.lock.Lock()
op.results[MemdService] = append(op.results[MemdService], EndpointPingResult{
Endpoint: serverAddress,
Error: err,
Latency: pingLatency,
Scope: op.bucketName,
ID: fmt.Sprintf("%p", pipeline),
State: state,
})
op.lock.Unlock()
wg.Done()
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdNoop,
Datatype: 0,
Cas: 0,
Key: nil,
Value: nil,
UserImpersonationFrame: userFrame,
},
Callback: handler,
RetryStrategy: retryStrat,
}
curOp, err := dc.kvMux.DispatchDirectToAddress(req, pipeline)
if err != nil {
op.lock.Lock()
op.results[MemdService] = append(op.results[MemdService], EndpointPingResult{
Endpoint: redactSystemData(serverAddress),
Error: err,
Latency: 0,
Scope: op.bucketName,
})
op.lock.Unlock()
wg.Done()
return
}
if !deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(deadline.Sub(start), func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallback(&TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "PingKV",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
})
}))
}
op.lock.Lock()
op.subops = append(op.subops, pingSubOp{
endpoint: serverAddress,
op: curOp,
})
op.lock.Unlock()
}(p)
// We iterate through all pipelines
return false
})
wg.Wait()
op.lock.Lock()
op.handledOneLocked(iter.RevID())
op.lock.Unlock()
return
}
select {
case <-ctx.Done():
ctxErr := ctx.Err()
var cancelReason error
if errors.Is(ctxErr, context.Canceled) {
cancelReason = ctxErr
} else {
cancelReason = errUnambiguousTimeout
}
op.lock.Lock()
op.results[MemdService] = append(op.results[MemdService], EndpointPingResult{
Error: cancelReason,
Scope: op.bucketName,
ID: uuid.New().String(),
State: PingStateTimeout,
})
op.handledOneLocked(iter.RevID())
op.lock.Unlock()
return
case <-time.After(interval):
}
}
}
func (dc *diagnosticsComponent) pingHTTP(ctx context.Context, service ServiceType,
interval time.Duration, deadline time.Time, retryStrat RetryStrategy, op *pingOp, ignoreMissingServices bool) {
if !deadline.IsZero() {
// We have to setup a new child context with its own deadline because services have their own timeout values.
var cancel context.CancelFunc
ctx, cancel = context.WithDeadline(ctx, deadline)
defer cancel()
}
muxer := dc.httpMux
var path string
switch service {
case N1qlService:
path = "/admin/ping"
case CbasService:
path = "/admin/ping"
case FtsService:
path = "/api/ping"
case CapiService:
path = "/"
}
for {
clientMux := muxer.Get()
if clientMux.revID > -1 {
var epList []routeEndpoint
switch service {
case N1qlService:
epList = clientMux.n1qlEpList
case CbasService:
epList = clientMux.cbasEpList
case FtsService:
epList = clientMux.ftsEpList
case MgmtService:
epList = clientMux.mgmtEpList
case CapiService:
epList = clientMux.capiEpList
}
if len(epList) == 0 {
op.lock.Lock()
if !ignoreMissingServices {
op.results[service] = append(op.results[service], EndpointPingResult{
Error: errServiceNotAvailable,
Scope: op.bucketName,
ID: uuid.New().String(),
})
}
op.handledOneLocked(clientMux.revID)
op.lock.Unlock()
return
}
var wg sync.WaitGroup
for _, ep := range epList {
wg.Add(1)
go func(ep string) {
defer wg.Done()
req := &httpRequest{
Service: service,
Method: "GET",
Path: path,
Endpoint: ep,
IsIdempotent: true,
RetryStrategy: retryStrat,
Context: ctx,
UniqueID: uuid.New().String(),
}
start := time.Now()
resp, err := dc.httpComponent.DoInternalHTTPRequest(req, false)
pingLatency := time.Since(start)
state := PingStateOK
if err != nil {
if errors.Is(err, ErrTimeout) {
state = PingStateTimeout
} else {
state = PingStateError
}
} else {
if resp.StatusCode > 200 {
state = PingStateError
b, pErr := ioutil.ReadAll(resp.Body)
if pErr != nil {
logDebugf("Failed to read response body for ping: %v", pErr)
}
err = errors.New(string(b))
}
}
op.lock.Lock()
op.results[service] = append(op.results[service], EndpointPingResult{
Endpoint: ep,
Error: err,
Latency: pingLatency,
Scope: op.bucketName,
ID: uuid.New().String(),
State: state,
})
op.lock.Unlock()
}(ep.Address)
}
wg.Wait()
op.lock.Lock()
op.handledOneLocked(clientMux.revID)
op.lock.Unlock()
return
}
select {
case <-ctx.Done():
ctxErr := ctx.Err()
var cancelReason error
if errors.Is(ctxErr, context.Canceled) {
cancelReason = ctxErr
} else {
cancelReason = errUnambiguousTimeout
}
op.lock.Lock()
op.results[service] = append(op.results[service], EndpointPingResult{
Error: cancelReason,
Scope: op.bucketName,
ID: uuid.New().String(),
State: PingStateTimeout,
})
op.handledOneLocked(clientMux.revID)
op.lock.Unlock()
return
case <-time.After(interval):
}
}
}
func (dc *diagnosticsComponent) Ping(opts PingOptions, cb PingCallback) (PendingOp, error) {
bucketName := ""
if dc.bucket != "" {
bucketName = redactMetaData(dc.bucket)
}
ignoreMissingServices := false
serviceTypes := opts.ServiceTypes
if len(serviceTypes) == 0 {
// We're defaulting to pinging what we can so don't ping anything that isn't in the cluster config
ignoreMissingServices = true
serviceTypes = []ServiceType{MemdService, CapiService, N1qlService, FtsService, CbasService, MgmtService}
}
ignoreMissingServices = ignoreMissingServices || opts.ignoreMissingServices
ctx, cancelFunc := context.WithCancel(context.Background())
op := &pingOp{
callback: cb,
remaining: int32(len(serviceTypes)),
results: make(map[ServiceType][]EndpointPingResult),
bucketName: bucketName,
httpCancel: cancelFunc,
}
retryStrat := newFailFastRetryStrategy()
// interval is how long to wait between checking if we've seen a cluster config
interval := 10 * time.Millisecond
for _, serviceType := range serviceTypes {
switch serviceType {
case MemdService:
go dc.pingKV(ctx, interval, opts.KVDeadline, retryStrat, opts.User, op)
case CapiService:
go dc.pingHTTP(ctx, CapiService, interval, opts.CapiDeadline, retryStrat, op, ignoreMissingServices)
case N1qlService:
go dc.pingHTTP(ctx, N1qlService, interval, opts.N1QLDeadline, retryStrat, op, ignoreMissingServices)
case FtsService:
go dc.pingHTTP(ctx, FtsService, interval, opts.FtsDeadline, retryStrat, op, ignoreMissingServices)
case CbasService:
go dc.pingHTTP(ctx, CbasService, interval, opts.CbasDeadline, retryStrat, op, ignoreMissingServices)
case MgmtService:
go dc.pingHTTP(ctx, MgmtService, interval, opts.MgmtDeadline, retryStrat, op, ignoreMissingServices)
}
}
return op, nil
}
// Diagnostics returns diagnostics information about the client.
// Mainly containing a list of open connections and their current
// states.
func (dc *diagnosticsComponent) Diagnostics(opts DiagnosticsOptions) (*DiagnosticInfo, error) {
for {
iter, err := dc.kvMux.PipelineSnapshot()
if err != nil {
return nil, err
}
var conns []MemdConnInfo
iter.Iterate(0, func(pipeline *memdPipeline) bool {
pipeline.clientsLock.Lock()
for _, pipecli := range pipeline.clients {
localAddr := ""
remoteAddr := ""
var lastActivity time.Time
pipecli.lock.Lock()
if pipecli.client != nil {
localAddr = pipecli.client.LocalAddress()
remoteAddr = pipecli.client.Address()
lastActivityUs := atomic.LoadInt64(&pipecli.client.lastActivity)
if lastActivityUs != 0 {
lastActivity = time.Unix(0, lastActivityUs)
}
}
pipecli.lock.Unlock()
conn := MemdConnInfo{
LocalAddr: localAddr,
RemoteAddr: remoteAddr,
LastActivity: lastActivity,
ID: fmt.Sprintf("%p", pipecli),
State: pipecli.State(),
}
if dc.bucket != "" {
conn.Scope = redactMetaData(dc.bucket)
}
conns = append(conns, conn)
}
pipeline.clientsLock.Unlock()
return false
})
expected := len(conns)
connected := 0
for _, conn := range conns {
if conn.State == EndpointStateConnected {
connected++
}
}
state := ClusterStateOffline
if connected == expected {
state = ClusterStateOnline
} else if connected > 1 {
state = ClusterStateDegraded
}
endIter, err := dc.kvMux.PipelineSnapshot()
if err != nil {
return nil, err
}
if iter.RevID() == endIter.RevID() {
return &DiagnosticInfo{
ConfigRev: iter.RevID(),
MemdConns: conns,
State: state,
}, nil
}
}
}
func (dc *diagnosticsComponent) checkKVReady(desiredState ClusterState, op *waitUntilOp) {
for {
iter, err := dc.kvMux.PipelineSnapshot()
if err != nil {
logErrorf("failed to get pipeline snapshot: %v", err)
shouldRetry, until := retryOrchMaybeRetry(op, NoPipelineSnapshotRetryReason)
if !shouldRetry {
op.cancel(err)
return
}
select {
case <-op.stopCh:
return
case <-time.After(time.Until(until)):
continue
}
}
var connectErr error
revID := iter.RevID()
if revID == -1 {
// We've not seen a config so let's see if we've been informed about any errors.
dc.preConfigBootstrapErrorLock.Lock()
connectErr = dc.preConfigBootstrapError
logDebugf("Bootstrap error found before config seen: %v", connectErr)
dc.preConfigBootstrapErrorLock.Unlock()
// If there's no error appearing from the pipeline client then let's check the poller
if connectErr == nil && dc.pollerErrorProvider != nil {
pollerErr := dc.pollerErrorProvider.PollerError()
// We don't care about timeouts, they don't tell us anything we want to know.
if pollerErr != nil && !errors.Is(pollerErr, ErrTimeout) {
logDebugf("Error found in poller before config seen: %v", pollerErr)
connectErr = pollerErr
}
}
if connectErr == nil {
logDebugf("No config seen yet in kv muxer but no errors found.")
}
} else if revID > -1 {
expected := iter.NumPipelines()
connected := 0
iter.Iterate(0, func(pipeline *memdPipeline) bool {
pipeline.clientsLock.Lock()
defer pipeline.clientsLock.Unlock()
for _, cli := range pipeline.clients {
state := cli.State()
if state == EndpointStateConnected {
connected++
if desiredState == ClusterStateDegraded {
// If we're after degraded state then we can just bail early as we've already fulfilled that.
return true
}
// We only need one of the pipeline clients to be connected for this pipeline to be considered
// online.
break
}
err := cli.Error()
if err != nil {
logDebugf("Error found in client after config seen: %v", err)
connectErr = err
// If the desired state is degraded then we need to keep trying as a different client or pipeline
// might be connected. If it's online then we can bail now as we'll never achieve that.
if desiredState == ClusterStateOnline {
return true
}
}
}
return false
})
// If there's no error appearing from the pipeline client then let's check the poller
if connectErr == nil && dc.pollerErrorProvider != nil {
pollerErr := dc.pollerErrorProvider.PollerError()
// We don't care about timeouts, they don't tell us anything we want to know.
if pollerErr != nil && !errors.Is(pollerErr, ErrTimeout) {
logDebugf("Error found in poller after config seen: %v", pollerErr)
connectErr = pollerErr
}
}
switch desiredState {
case ClusterStateDegraded:
if connected > 0 {
op.lock.Lock()
op.handledOneLocked()
op.lock.Unlock()
return
}
case ClusterStateOnline:
if connected == expected {
op.lock.Lock()
op.handledOneLocked()
op.lock.Unlock()
return
}
default:
// How we got here no-one does know
// But round and round we must go
}
}
var until time.Time
if connectErr == nil {
var shouldRetry bool
shouldRetry, until = retryOrchMaybeRetry(op, NotReadyRetryReason)
if !shouldRetry {
op.cancel(errCliInternalError)
return
}
} else {
var shouldRetry bool
if errors.Is(connectErr, ErrBucketNotFound) {
shouldRetry, until = retryOrchMaybeRetry(op, BucketNotReadyReason)
} else {
shouldRetry, until = retryOrchMaybeRetry(op, ConnectionErrorRetryReason)
}
if !shouldRetry {
op.cancel(connectErr)
return
}
}
select {
case <-op.stopCh:
return
case <-time.After(time.Until(until)):
}
}
}
func (dc *diagnosticsComponent) checkHTTPReady(ctx context.Context, service ServiceType,
desiredState ClusterState, forceWait bool, op *waitUntilOp) {
retryStrat := &failFastRetryStrategy{}
muxer := dc.httpMux
var path string
switch service {
case N1qlService:
path = "/admin/ping"
case CbasService:
path = "/admin/ping"
case FtsService:
path = "/api/ping"
case CapiService:
path = "/"
case MgmtService:
path = ""
}
for {
clientMux := muxer.Get()
var connectErr error
if clientMux.revID == -1 {
// We've not seen a config so let's see if we've been informed about any errors.
dc.preConfigBootstrapErrorLock.Lock()
connectErr = dc.preConfigBootstrapError
logDebugf("Bootstrap error found before config seen: %v", connectErr)
dc.preConfigBootstrapErrorLock.Unlock()
// If there's no error appearing from the pipeline client then let's check the poller
if connectErr == nil && dc.pollerErrorProvider != nil {
pollerErr := dc.pollerErrorProvider.PollerError()
// We don't care about timeouts, they don't tell us anything we want to know.
if pollerErr != nil && !errors.Is(pollerErr, ErrTimeout) {
logDebugf("Error found in poller before config seen: %v", pollerErr)
connectErr = pollerErr
}
}
if connectErr == nil {
logDebugf("No config seen yet in http muxer but no errors found.")
}
} else {
var epList []routeEndpoint
switch service {
case N1qlService:
epList = clientMux.n1qlEpList
case CbasService:
epList = clientMux.cbasEpList
case FtsService:
epList = clientMux.ftsEpList
case CapiService:
epList = clientMux.capiEpList
case MgmtService:
epList = clientMux.mgmtEpList
}
connected := uint32(0)
func() {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var wg sync.WaitGroup
for _, ep := range epList {
wg.Add(1)
go func(ep string) {
defer wg.Done()
req := &httpRequest{
Service: service,
Method: "GET",
Path: path,
RetryStrategy: retryStrat,
Endpoint: ep,
IsIdempotent: true,
Context: ctx,
UniqueID: uuid.New().String(),
}
resp, err := dc.httpComponent.DoInternalHTTPRequest(req, false)
if err != nil {
if errors.Is(err, context.Canceled) {
return
}
logDebugf("Error returned for HTTP request for service %d: %v", service, err)
if desiredState == ClusterStateOnline {
// Cancel this run entirely, we can't satisfy the requirements
cancel()
}
return
}
if resp.StatusCode != 200 {
logDebugf("Non-200 status code returned for HTTP request for service %d: %d", service, resp.StatusCode)
if desiredState == ClusterStateOnline {
// Cancel this run entirely, we can't satisfy the requirements
cancel()
}
return
}
atomic.AddUint32(&connected, 1)
if desiredState == ClusterStateDegraded {
// Cancel this run entirely, we've successfully satisfied the requirements
cancel()
}
}(ep.Address)
}
wg.Wait()
}()
switch desiredState {
case ClusterStateDegraded:
if atomic.LoadUint32(&connected) > 0 {
op.lock.Lock()
op.handledOneLocked()
op.lock.Unlock()
return
}
case ClusterStateOnline:
if !forceWait && len(epList) == 0 {
op.lock.Lock()
op.handledOneLocked()
op.lock.Unlock()
return
}
// If there are no entries in the epList then the service is not online and so cannot be ready.
if len(epList) > 0 && atomic.LoadUint32(&connected) == uint32(len(epList)) {
op.lock.Lock()
op.handledOneLocked()
op.lock.Unlock()
return
}
default:
// How we got here no-one does know
// But round and round we must go
}
}
var until time.Time
if connectErr == nil {
var shouldRetry bool
shouldRetry, until = retryOrchMaybeRetry(op, NotReadyRetryReason)
if !shouldRetry {
op.cancel(errCliInternalError)
return
}
} else {
var shouldRetry bool
if errors.Is(connectErr, ErrBucketNotFound) {
shouldRetry, until = retryOrchMaybeRetry(op, BucketNotReadyReason)
} else {
shouldRetry, until = retryOrchMaybeRetry(op, ConnectionErrorRetryReason)
}
if !shouldRetry {
op.cancel(connectErr)
return
}
}
select {
case <-op.stopCh:
return
case <-time.After(time.Until(until)):
}
}
}
func (dc *diagnosticsComponent) WaitUntilReady(deadline time.Time, forceWait bool, opts WaitUntilReadyOptions,
cb WaitUntilReadyCallback) (PendingOp, error) {
desiredState := opts.DesiredState
if desiredState == ClusterStateOffline {
return nil, wrapError(errInvalidArgument, "cannot use offline as a desired state")
}
if desiredState == 0 {
desiredState = ClusterStateOnline
}
retry := opts.RetryStrategy
if retry == nil {
retry = dc.defaultRetry
}
ctx, cancelFunc := context.WithCancel(context.Background())
op := &waitUntilOp{
remaining: int32(len(opts.ServiceTypes)),
stopCh: make(chan struct{}),
callback: cb,
httpCancel: cancelFunc,
retryStrat: retry,
}
op.lock.Lock()
start := time.Now()
op.timer = time.AfterFunc(deadline.Sub(start), func() {
op.cancel(&TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "WaitUntilReady",
TimeObserved: time.Since(start),
RetryReasons: op.RetryReasons(),
RetryAttempts: op.RetryAttempts(),
})
})
op.lock.Unlock()
for _, serviceType := range opts.ServiceTypes {
switch serviceType {
case MemdService:
go dc.checkKVReady(desiredState, op)
case CapiService:
go dc.checkHTTPReady(ctx, CapiService, desiredState, forceWait, op)
case N1qlService:
go dc.checkHTTPReady(ctx, N1qlService, desiredState, forceWait, op)
case FtsService:
go dc.checkHTTPReady(ctx, FtsService, desiredState, forceWait, op)
case CbasService:
go dc.checkHTTPReady(ctx, CbasService, desiredState, forceWait, op)
case MgmtService:
go dc.checkHTTPReady(ctx, MgmtService, desiredState, forceWait, op)
}
}
return op, nil
}
gocbcore-10.2.3/dyntlsconfig.go 0000664 0000000 0000000 00000001625 14417540156 0016425 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"crypto/tls"
"crypto/x509"
"net"
)
type dynTLSConfig struct {
BaseConfig *tls.Config
Provider func() *x509.CertPool
}
func (config dynTLSConfig) Clone() *dynTLSConfig {
return &dynTLSConfig{
BaseConfig: config.BaseConfig.Clone(),
Provider: config.Provider,
}
}
func (config dynTLSConfig) MakeForHost(serverName string) (*tls.Config, error) {
newConfig := config.BaseConfig.Clone()
if config.Provider != nil {
rootCAs := config.Provider()
if rootCAs != nil {
newConfig.RootCAs = rootCAs
newConfig.InsecureSkipVerify = false
} else {
newConfig.RootCAs = nil
newConfig.InsecureSkipVerify = true
}
}
newConfig.ServerName = serverName
return newConfig, nil
}
func (config dynTLSConfig) MakeForAddr(addr string) (*tls.Config, error) {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
return config.MakeForHost(host)
}
gocbcore-10.2.3/errmap.go 0000664 0000000 0000000 00000005463 14417540156 0015214 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"strconv"
"time"
)
type kvErrorMapAttribute string
type kvErrorMapRetry struct {
Strategy string
Interval int
After int
Ceil int
MaxDuration int
}
func (retry kvErrorMapRetry) CalculateRetryDelay(retryCount uint32) time.Duration {
duraCeil := time.Duration(retry.Ceil) * time.Millisecond
var dura time.Duration
if retryCount == 0 {
dura = time.Duration(retry.After) * time.Millisecond
} else {
interval := time.Duration(retry.Interval) * time.Millisecond
if retry.Strategy == "constant" {
dura = interval
} else if retry.Strategy == "linear" {
dura = interval * time.Duration(retryCount)
} else if retry.Strategy == "exponential" {
dura = interval
for i := uint32(0); i < retryCount-1; i++ {
// Need to multiply by the original value, not the scaled one
dura = dura * time.Duration(retry.Interval)
// We have to check this here to make sure we do not overflow
if duraCeil > 0 && dura > duraCeil {
dura = duraCeil
break
}
}
}
}
if duraCeil > 0 && dura > duraCeil {
dura = duraCeil
}
return dura
}
type kvErrorMapError struct {
Name string
Description string
Attributes []kvErrorMapAttribute
Retry kvErrorMapRetry
}
type kvErrorMap struct {
Version int
Revision int
Errors map[uint16]kvErrorMapError
}
type cfgKvErrorMapError struct {
Name string `json:"name"`
Desc string `json:"desc"`
Attrs []string `json:"attrs"`
Retry struct {
Strategy string `json:"strategy"`
Interval int `json:"interval"`
After int `json:"after"`
Ceil int `json:"ceil"`
MaxDuration int `json:"max-duration"`
} `json:"retry"`
}
type cfgKvErrorMap struct {
Version int `json:"version"`
Revision int `json:"revision"`
Errors map[string]cfgKvErrorMapError
}
func parseKvErrorMap(data []byte) (*kvErrorMap, error) {
var cfg cfgKvErrorMap
if err := json.Unmarshal(data, &cfg); err != nil {
return nil, err
}
var errMap kvErrorMap
errMap.Version = cfg.Version
errMap.Revision = cfg.Revision
errMap.Errors = make(map[uint16]kvErrorMapError)
for errCodeStr, errData := range cfg.Errors {
errCode, err := strconv.ParseInt(errCodeStr, 16, 64)
if err != nil {
return nil, err
}
var errInfo kvErrorMapError
errInfo.Name = errData.Name
errInfo.Description = errData.Desc
errInfo.Attributes = make([]kvErrorMapAttribute, len(errData.Attrs))
for i, attr := range errData.Attrs {
errInfo.Attributes[i] = kvErrorMapAttribute(attr)
}
errInfo.Retry.Strategy = errData.Retry.Strategy
errInfo.Retry.Interval = errData.Retry.Interval
errInfo.Retry.After = errData.Retry.After
errInfo.Retry.Ceil = errData.Retry.Ceil
errInfo.Retry.MaxDuration = errData.Retry.MaxDuration
errMap.Errors[uint16(errCode)] = errInfo
}
return &errMap, nil
}
gocbcore-10.2.3/errmap_test.go 0000664 0000000 0000000 00000013226 14417540156 0016247 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"github.com/couchbase/gocbcore/v10/memd"
"testing"
"time"
)
func TestKvErrorConstantRetry(t *testing.T) {
constant := kvErrorMapRetry{
Strategy: "constant",
Interval: 1000,
After: 2000,
Ceil: 4000,
MaxDuration: 3000,
}
if constant.CalculateRetryDelay(0) != 2000*time.Millisecond {
t.Fatalf("failed to respect after for first retry")
}
if constant.CalculateRetryDelay(1) != 1000*time.Millisecond {
t.Fatalf("should respect interval for second retry")
}
if constant.CalculateRetryDelay(3) != 1000*time.Millisecond {
t.Fatalf("should respect interval for minimal retries")
}
if constant.CalculateRetryDelay(15000) != 1000*time.Millisecond {
t.Fatalf("should respect interval for large retry counts")
}
}
func TestKvErrorLinearRetry(t *testing.T) {
linear := kvErrorMapRetry{
Strategy: "linear",
Interval: 1000,
After: 2000,
Ceil: 60000,
MaxDuration: 100000,
}
if linear.CalculateRetryDelay(0) != 2000*time.Millisecond {
t.Fatalf("failed to respect after for first retry")
}
if linear.CalculateRetryDelay(1) != 1000*time.Millisecond {
t.Fatalf("should respect interval for second retry")
}
if linear.CalculateRetryDelay(3) != 3000*time.Millisecond {
t.Fatalf("should respect interval for minimal retries")
}
if linear.CalculateRetryDelay(150) != 60000*time.Millisecond {
t.Fatalf("should respect ceiling for large retry counts")
}
}
func TestKvErrorExponentialRetry(t *testing.T) {
exponential := kvErrorMapRetry{
Strategy: "exponential",
Interval: 10,
After: 1000,
Ceil: 60000,
MaxDuration: 100000,
}
if exponential.CalculateRetryDelay(0) != 1000*time.Millisecond {
t.Fatalf("failed to respect after for first retry")
}
if exponential.CalculateRetryDelay(1) != 10*time.Millisecond {
t.Fatalf("should respect interval for second retry")
}
if exponential.CalculateRetryDelay(3) != 1000*time.Millisecond {
t.Fatalf("should respect interval for minimal retries")
}
if exponential.CalculateRetryDelay(400) != 60000*time.Millisecond {
t.Fatalf("should respect ceiling for large retry counts")
}
}
type errMapTestRetryStrategy struct {
reasons []RetryReason
retries int
}
func (lrs *errMapTestRetryStrategy) RetryAfter(request RetryRequest, reason RetryReason) RetryAction {
lrs.retries++
lrs.reasons = append(lrs.reasons, reason)
return &WithDurationRetryAction{50 * time.Millisecond}
}
func (suite *StandardTestSuite) testKvErrorMapGeneric(checkName TestName) {
suite.EnsureSupportsFeature(TestFeatureErrMap)
if !suite.IsMockServer() {
suite.T().Skipf("only supported when testing against mock server")
}
testKey := "hello"
spec := suite.StartTest(checkName)
h := suite.GetHarness()
agent := spec.Agent
strategy := &errMapTestRetryStrategy{}
h.PushOp(agent.Get(GetOptions{
Key: []byte(testKey),
RetryStrategy: strategy,
CollectionName: spec.Collection,
ScopeName: spec.Scope,
}, func(res *GetResult, err error) {
h.Wrap(func() {})
}))
h.Wait(0)
if strategy.retries != 3 {
suite.T().Fatalf("Expected retries to be 3 but was %d", strategy.retries)
}
if len(strategy.reasons) != 3 {
suite.T().Fatalf("Expected 3 retry reasons but was %v", strategy.reasons)
}
for _, reason := range strategy.reasons {
if reason != KVErrMapRetryReason {
suite.T().Fatalf("Expected reason to be KVErrMapRetryReason but was %s", reason.Description())
}
}
suite.VerifyKVMetrics(spec.Meter, "Get", 1, false, false)
suite.EndTest(spec)
}
// It doesn't actually matter what strategy the error map specifies, we just test that retries happen as the
// strategy dictates no matter what.
func (suite *StandardTestSuite) TestKvErrorMap7ff0() {
suite.testKvErrorMapGeneric(TestNameErrMapLinearRetry)
}
func (suite *StandardTestSuite) TestKvErrorMap7ff1() {
suite.testKvErrorMapGeneric(TestNameErrMapConstantRetry)
}
func (suite *StandardTestSuite) TestKvErrorMap7ff2() {
suite.testKvErrorMapGeneric(TestNameErrMapExponentialRetry)
}
func (suite *UnitTestSuite) TestStoreKVErrorMapV1() {
data, err := loadRawTestDataset("err_map70_v1")
suite.Require().Nil(err, err)
errMgr := newErrMapManager("test")
errMgr.StoreErrorMap(data)
errMap := errMgr.kvErrorMap.Get()
suite.Require().NotNil(errMap)
suite.Assert().Equal(1, errMap.Version)
suite.Assert().Equal(2, errMap.Revision)
suite.Assert().Len(errMap.Errors, 58)
entry := errMgr.getKvErrMapData(memd.StatusLocked)
suite.Require().NotNil(entry)
suite.Assert().Equal("LOCKED", entry.Name)
suite.Assert().Equal("Requested resource is locked", entry.Description)
suite.Assert().Len(entry.Attributes, 3)
suite.Assert().Contains(entry.Attributes, kvErrorMapAttribute("item-locked"))
suite.Assert().Contains(entry.Attributes, kvErrorMapAttribute("item-only"))
suite.Assert().Contains(entry.Attributes, kvErrorMapAttribute("retry-now"))
}
func (suite *UnitTestSuite) TestStoreKVErrorMapV2() {
data, err := loadRawTestDataset("err_map71_v2")
suite.Require().Nil(err, err)
errMgr := newErrMapManager("test")
errMgr.StoreErrorMap(data)
errMap := errMgr.kvErrorMap.Get()
suite.Require().NotNil(errMap)
suite.Assert().Equal(2, errMap.Version)
suite.Assert().Equal(1, errMap.Revision)
suite.Assert().Len(errMap.Errors, 65)
entry := errMgr.getKvErrMapData(memd.StatusLocked)
suite.Require().NotNil(entry)
suite.Assert().Equal("LOCKED", entry.Name)
suite.Assert().Equal("Requested resource is locked", entry.Description)
suite.Assert().Len(entry.Attributes, 3)
suite.Assert().Contains(entry.Attributes, kvErrorMapAttribute("item-locked"))
suite.Assert().Contains(entry.Attributes, kvErrorMapAttribute("item-only"))
suite.Assert().Contains(entry.Attributes, kvErrorMapAttribute("retry-now"))
}
gocbcore-10.2.3/errmapcomponent.go 0000664 0000000 0000000 00000014436 14417540156 0017137 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"github.com/couchbase/gocbcore/v10/memd"
)
type errMapComponent struct {
kvErrorMap kvErrorMapPtr
bucketName string
}
func newErrMapManager(bucketName string) *errMapComponent {
return &errMapComponent{
bucketName: bucketName,
}
}
func (errMgr *errMapComponent) getKvErrMapData(code memd.StatusCode) *kvErrorMapError {
errMap := errMgr.kvErrorMap.Get()
if errMap != nil {
if errData, ok := errMap.Errors[uint16(code)]; ok {
return &errData
}
}
return nil
}
func (errMgr *errMapComponent) StoreErrorMap(mapBytes []byte) {
errMap, err := parseKvErrorMap(mapBytes)
if err != nil {
logDebugf("Failed to parse kv error map (%s)", err)
return
}
logDebugf("Fetched error map: %+v", errMap)
// Check if we need to switch the agent itself to a better
// error map revision.
for {
origMap := errMgr.kvErrorMap.Get()
if origMap != nil && errMap.Revision < origMap.Revision {
break
}
if errMgr.kvErrorMap.Update(origMap, errMap) {
break
}
}
}
func (errMgr *errMapComponent) ShouldRetry(status memd.StatusCode) bool {
kvErrData := errMgr.getKvErrMapData(status)
if kvErrData != nil {
for _, attr := range kvErrData.Attributes {
if attr == "auto-retry" || attr == "retry-now" || attr == "retry-later" {
return true
}
}
}
return false
}
func (errMgr *errMapComponent) EnhanceKvError(err error, resp *memdQResponse, req *memdQRequest) error {
enhErr := &KeyValueError{
InnerError: err,
}
if req != nil {
enhErr.DocumentKey = string(req.Key)
enhErr.BucketName = errMgr.bucketName
enhErr.ScopeName = req.ScopeName
enhErr.CollectionName = req.CollectionName
enhErr.CollectionID = req.CollectionID
retryCount, reasons := req.Retries()
enhErr.RetryReasons = reasons
enhErr.RetryAttempts = retryCount
connInfo := req.ConnectionInfo()
enhErr.LastDispatchedTo = connInfo.lastDispatchedTo
enhErr.LastDispatchedFrom = connInfo.lastDispatchedFrom
enhErr.LastConnectionID = connInfo.lastConnectionID
enhErr.Internal.ResourceUnits = req.ResourceUnits()
}
if resp != nil {
enhErr.StatusCode = resp.Status
enhErr.Opaque = resp.Opaque
errMapData := errMgr.getKvErrMapData(enhErr.StatusCode)
if errMapData != nil {
enhErr.ErrorName = errMapData.Name
enhErr.ErrorDescription = errMapData.Description
}
if memd.DatatypeFlag(resp.Datatype)&memd.DatatypeFlagJSON != 0 {
var enhancedData struct {
Error struct {
Context string `json:"context"`
Ref string `json:"ref"`
} `json:"error"`
}
if parseErr := json.Unmarshal(resp.Value, &enhancedData); parseErr == nil {
enhErr.Context = enhancedData.Error.Context
enhErr.Ref = enhancedData.Error.Ref
}
}
}
return enhErr
}
func translateMemdError(err error, req *memdQRequest) error {
switch err {
case ErrMemdInvalidArgs:
return errInvalidArgument
case ErrMemdInternalError:
return errInternalServerFailure
case ErrMemdAccessError:
return errAuthenticationFailure
case ErrMemdAuthError:
return errAuthenticationFailure
case ErrMemdTmpFail:
return errTemporaryFailure
case ErrMemdBusy:
return errTemporaryFailure
case ErrMemdKeyExists:
if req.Command == memd.CmdReplace || (req.Command == memd.CmdDelete && req.Cas != 0) ||
(req.Command == memd.CmdSubDocMultiMutation && req.Cas != 0) {
return errCasMismatch
}
return errDocumentExists
case ErrMemdNotStored:
// GOCBC-1356: memcached does not currently return a NOT_STORED response when inserting a doc, but this was originally
// the plan, so for safety handle this path.
if req.Command == memd.CmdAdd {
return errDocumentExists
}
return errNotStored
case ErrMemdCollectionNotFound:
return errCollectionNotFound
case ErrMemdUnknownCommand:
return errUnsupportedOperation
case ErrMemdNotSupported:
return errUnsupportedOperation
case ErrMemdDCPStreamIDInvalid:
return errDCPStreamIDInvalid
case ErrMemdKeyNotFound:
return errDocumentNotFound
case ErrMemdLocked:
// BUGFIX(brett19): This resolves a bug in the server processing of the LOCKED
// operation where the server will respond with LOCKED rather than a CAS mismatch.
if req.Command == memd.CmdUnlockKey {
return errCasMismatch
}
return errDocumentLocked
case ErrMemdTooBig:
return errValueTooLarge
case ErrMemdSubDocNotJSON:
return errValueNotJSON
case ErrMemdDurabilityInvalidLevel:
return errDurabilityLevelNotAvailable
case ErrMemdDurabilityImpossible:
return errDurabilityImpossible
case ErrMemdSyncWriteAmbiguous:
return errDurabilityAmbiguous
case ErrMemdSyncWriteInProgess:
return errDurableWriteInProgress
case ErrMemdSyncWriteReCommitInProgress:
return errDurableWriteReCommitInProgress
case ErrMemdSubDocPathNotFound:
return errPathNotFound
case ErrMemdSubDocPathInvalid:
return errPathInvalid
case ErrMemdSubDocPathTooBig:
return errPathTooBig
case ErrMemdSubDocDocTooDeep:
return errPathTooDeep
case ErrMemdSubDocValueTooDeep:
return errValueTooDeep
case ErrMemdSubDocCantInsert:
return errValueInvalid
case ErrMemdSubDocNotJSON:
return errDocumentNotJSON
case ErrMemdSubDocBadRange:
return errNumberTooBig
case ErrMemdBadDelta:
return errDeltaInvalid
case ErrMemdSubDocBadDelta:
return errDeltaInvalid
case ErrMemdSubDocPathExists:
return errPathExists
case ErrXattrUnknownMacro:
return errXattrUnknownMacro
case ErrXattrInvalidFlagCombo:
return errXattrInvalidFlagCombo
case ErrXattrInvalidKeyCombo:
return errXattrInvalidKeyCombo
case ErrMemdSubDocXattrUnknownVAttr:
return errXattrUnknownVirtualAttribute
case ErrMemdSubDocXattrCannotModifyVAttr:
return errXattrCannotModifyVirtualAttribute
case ErrXattrInvalidOrder:
return errXattrInvalidOrder
case ErrMemdNotMyVBucket:
return errNotMyVBucket
case ErrMemdRateLimitedNetworkIngress:
return errRateLimitedFailure
case ErrMemdRateLimitedNetworkEgress:
return errRateLimitedFailure
case ErrMemdRateLimitedMaxConnections:
return errRateLimitedFailure
case ErrMemdRateLimitedMaxCommands:
return errRateLimitedFailure
case ErrMemdRateLimitedScopeSizeLimitExceeded:
return errQuotaLimitedFailure
case ErrMemdRangeScanCancelled:
return errRangeScanCancelled
case ErrMemdRangeScanMore:
return errRangeScanMore
case ErrMemdRangeScanComplete:
return errRangeScanComplete
case ErrMemdRangeScanVbUUIDNotEqual:
return errRangeScanVbUUIDNotEqual
}
return err
}
gocbcore-10.2.3/errmapptr.go 0000664 0000000 0000000 00000000704 14417540156 0015733 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"sync/atomic"
"unsafe"
)
type kvErrorMapPtr struct {
data unsafe.Pointer
}
func (ptr *kvErrorMapPtr) Get() *kvErrorMap {
return (*kvErrorMap)(atomic.LoadPointer(&ptr.data))
}
func (ptr *kvErrorMapPtr) Update(old, new *kvErrorMap) bool {
if new == nil {
logErrorf("Attempted to update to nil kvErrorMap")
return false
}
return atomic.CompareAndSwapPointer(&ptr.data, unsafe.Pointer(old), unsafe.Pointer(new))
}
gocbcore-10.2.3/error.go 0000664 0000000 0000000 00000023574 14417540156 0015062 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"io"
)
// dwError is a special error used for the purposes of rewrapping
// another error to provide more detailed information inherently
// with the error type itself. Mainly used for timeout and rate limiting.
type dwError struct {
InnerError error
Message string
}
func (e dwError) Error() string {
return e.Message
}
func (e dwError) Unwrap() error {
return e.InnerError
}
var (
// ErrNoSupportedMechanisms occurs when the server does not support any of the
// authentication methods that the client finds suitable.
ErrNoSupportedMechanisms = errors.New("no supported authentication mechanisms")
// ErrBadHosts occurs when the list of hosts specified cannot be contacted.
ErrBadHosts = errors.New("failed to connect to any of the specified hosts")
// ErrProtocol occurs when the server responds with unexpected or unparseable data.
ErrProtocol = errors.New("failed to parse server response")
// ErrNoReplicas occurs when no replicas respond in time
ErrNoReplicas = errors.New("no replicas responded in time")
// ErrCliInternalError indicates an internal error occurred within the client.
ErrCliInternalError = errors.New("client internal error")
// ErrInvalidCredentials is returned when an invalid set of credentials is provided for a service.
ErrInvalidCredentials = errors.New("an invalid set of credentials was provided")
// ErrInvalidServer occurs when an explicit, but invalid server is specified.
ErrInvalidServer = errors.New("specific server is invalid")
// ErrInvalidVBucket occurs when an explicit, but invalid vbucket index is specified.
ErrInvalidVBucket = errors.New("specific vbucket index is invalid")
// ErrInvalidReplica occurs when an explicit, but invalid replica index is specified.
ErrInvalidReplica = errors.New("specific server index is invalid")
// ErrInvalidService occurs when an explicit but invalid service type is specified
ErrInvalidService = errors.New("invalid service")
// ErrInvalidCertificate occurs when a certificate that is not useable is passed to an Agent.
ErrInvalidCertificate = errors.New("certificate is invalid")
// ErrCollectionsUnsupported occurs when collections are used but either server does not support them or the agent
// was created without them enabled.
ErrCollectionsUnsupported = errors.New("collections are not enabled")
// ErrBucketAlreadySelected occurs when SelectBucket is called when a bucket is already selected..
ErrBucketAlreadySelected = errors.New("bucket already selected")
// ErrShutdown occurs when operations are performed on a previously closed Agent.
ErrShutdown = errors.New("connection shut down")
// ErrOverload occurs when too many operations are dispatched and all queues are full.
ErrOverload = errors.New("queue overflowed")
// ErrSocketClosed occurs when a socket closes while an operation is in flight.
ErrSocketClosed = io.EOF
// ErrGCCCPInUse occurs when an operation dis performed whilst the client is connect via GCCCP.
ErrGCCCPInUse = errors.New("connected via gcccp, kv operations are not supported, open a bucket first")
// ErrNotMyVBucket occurs when an operation is sent to a node which does not own the vbucket.
ErrNotMyVBucket = errors.New("not my vbucket")
// ErrForcedReconnect occurs when an operation is in flight during a forced reconnect.
ErrForcedReconnect = errors.New("forced reconnect")
// ErrNotStored occurs when the server could not store the document.
// Per GOCBC-1356, it can also be returned on some paths when inserting a document, and in that context indicates
// that the document already exists.
ErrNotStored = errors.New("document was not stored")
)
// Shared Error Definitions RFC#58@15
var (
// ErrTimeout occurs when an operation does not receive a response in a timely manner.
ErrTimeout = errors.New("operation has timed out")
ErrRequestCanceled = errors.New("request canceled")
ErrInvalidArgument = errors.New("invalid argument")
ErrServiceNotAvailable = errors.New("service not available")
ErrInternalServerFailure = errors.New("internal server failure")
ErrAuthenticationFailure = errors.New("authentication failure - possible reasons - incorrect authentication configuration, bucket doesn’t exist or bucket may be hibernated")
ErrTemporaryFailure = errors.New("temporary failure")
ErrParsingFailure = errors.New("parsing failure")
ErrMemdClientClosed = errors.New("memdclient closed")
ErrRequestAlreadyDispatched = errors.New("request already dispatched")
ErrCasMismatch = errors.New("cas mismatch")
ErrBucketNotFound = errors.New("bucket not found")
ErrCollectionNotFound = errors.New("collection not found")
ErrEncodingFailure = errors.New("encoding failure")
ErrDecodingFailure = errors.New("decoding failure")
ErrUnsupportedOperation = errors.New("unsupported operation")
ErrAmbiguousTimeout = &dwError{ErrTimeout, "ambiguous timeout"}
ErrUnambiguousTimeout = &dwError{ErrTimeout, "unambiguous timeout"}
// ErrFeatureNotAvailable occurs when an operation is performed on a bucket which does not support it.
ErrFeatureNotAvailable = errors.New("feature is not available")
ErrScopeNotFound = errors.New("scope not found")
ErrIndexNotFound = errors.New("index not found")
ErrIndexExists = errors.New("index exists")
// Uncommitted: This API may change in the future.
ErrRateLimitedFailure = errors.New("rate limited failure")
// Uncommitted: This API may change in the future.
ErrQuotaLimitedFailure = errors.New("quota limited failure")
)
// Key Value Error Definitions RFC#58@15
var (
ErrDocumentNotFound = errors.New("document not found")
ErrDocumentUnretrievable = errors.New("document unretrievable")
ErrDocumentLocked = errors.New("document locked")
ErrValueTooLarge = errors.New("value too large")
ErrDocumentExists = errors.New("document exists")
ErrValueNotJSON = errors.New("value not json")
ErrDurabilityLevelNotAvailable = errors.New("durability level not available")
ErrDurabilityImpossible = errors.New("durability impossible")
ErrDurabilityAmbiguous = errors.New("durability ambiguous")
ErrDurableWriteInProgress = errors.New("durable write in progress")
ErrDurableWriteReCommitInProgress = errors.New("durable write recommit in progress")
ErrMutationLost = errors.New("mutation lost")
ErrPathNotFound = errors.New("path not found")
ErrPathMismatch = errors.New("path mismatch")
ErrPathInvalid = errors.New("path invalid")
ErrPathTooBig = errors.New("path too big")
ErrPathTooDeep = errors.New("path too deep")
ErrValueTooDeep = errors.New("value too deep")
ErrValueInvalid = errors.New("value invalid")
ErrDocumentNotJSON = errors.New("document not json")
ErrNumberTooBig = errors.New("number too big")
ErrDeltaInvalid = errors.New("delta invalid")
ErrPathExists = errors.New("path exists")
ErrXattrUnknownMacro = errors.New("xattr unknown macro")
ErrXattrInvalidFlagCombo = errors.New("xattr invalid flag combination")
ErrXattrInvalidKeyCombo = errors.New("xattr invalid key combination")
ErrXattrUnknownVirtualAttribute = errors.New("xattr unknown virtual attribute")
ErrXattrCannotModifyVirtualAttribute = errors.New("xattr cannot modify virtual attribute")
ErrXattrInvalidOrder = errors.New("xattr invalid order")
ErrRangeScanCancelled = errors.New("range scan cancelled")
ErrRangeScanMore = errors.New("range scan more")
ErrRangeScanComplete = errors.New("range scan complete")
ErrRangeScanVbUUIDNotEqual = errors.New("range scan vb-uuid mismatch")
)
// Query Error Definitions RFC#58@15
var (
ErrPlanningFailure = errors.New("planning failure")
ErrIndexFailure = errors.New("index failure")
ErrPreparedStatementFailure = errors.New("prepared statement failure")
ErrDMLFailure = errors.New("data service returned an error during execution of DML statement")
)
// Analytics Error Definitions RFC#58@15
var (
ErrCompilationFailure = errors.New("compilation failure")
ErrJobQueueFull = errors.New("job queue full")
ErrDatasetNotFound = errors.New("dataset not found")
ErrDataverseNotFound = errors.New("dataverse not found")
ErrDatasetExists = errors.New("dataset exists")
ErrDataverseExists = errors.New("dataverse exists")
ErrLinkNotFound = errors.New("link not found")
)
// Search Error Definitions RFC#58@15
var ()
// View Error Definitions RFC#58@15
var (
ErrViewNotFound = errors.New("view not found")
ErrDesignDocumentNotFound = errors.New("design document not found")
)
// Management Error Definitions RFC#58@15
var (
ErrCollectionExists = errors.New("collection exists")
ErrScopeExists = errors.New("scope exists")
ErrUserNotFound = errors.New("user not found")
ErrGroupNotFound = errors.New("group not found")
ErrBucketExists = errors.New("bucket exists")
ErrUserExists = errors.New("user exists")
ErrBucketNotFlushable = errors.New("bucket not flushable")
ErrEventingFunctionNotFound = errors.New("eventing function not found")
ErrEventingFunctionNotDeployed = errors.New("eventing function not deployed")
ErrEventingFunctionCompilationFailure = errors.New("eventing function compilation failure")
ErrEventingFunctionIdenticalKeyspace = errors.New("eventing function identical keyspace")
ErrEventingFunctionNotBootstrapped = errors.New("eventing function not bootstrapped")
ErrEventingFunctionNotUndeployed = errors.New("eventing function not undeployed")
)
gocbcore-10.2.3/error_dcp.go 0000664 0000000 0000000 00000003760 14417540156 0015703 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"log"
"github.com/couchbase/gocbcore/v10/memd"
)
var streamEndErrorMap = make(map[memd.StreamEndStatus]error)
func makeStreamEndStatusError(code memd.StreamEndStatus) error {
err := errors.New(code.KVText())
if streamEndErrorMap[code] != nil {
log.Fatal("error handling setup failure")
}
streamEndErrorMap[code] = err
return err
}
func getStreamEndStatusError(code memd.StreamEndStatus) error {
if code == memd.StreamEndOK {
return nil
}
if err := streamEndErrorMap[code]; err != nil {
return err
}
return errors.New(code.KVText())
}
var (
// ErrDCPStreamClosed occurs when a DCP stream is closed gracefully.
ErrDCPStreamClosed = makeStreamEndStatusError(memd.StreamEndClosed)
// ErrDCPStreamStateChanged occurs when a DCP stream is interrupted by failover.
ErrDCPStreamStateChanged = makeStreamEndStatusError(memd.StreamEndStateChanged)
// ErrDCPStreamDisconnected occurs when a DCP stream is disconnected.
ErrDCPStreamDisconnected = makeStreamEndStatusError(memd.StreamEndDisconnected)
// ErrDCPStreamTooSlow occurs when a DCP stream is cancelled due to the application
// not keeping up with the rate of flow of DCP events sent by the server.
ErrDCPStreamTooSlow = makeStreamEndStatusError(memd.StreamEndTooSlow)
// ErrDCPBackfillFailed occurs when there was an issue starting the backfill on
// the server e.g. the requested start seqno was behind the purge seqno.
ErrDCPBackfillFailed = makeStreamEndStatusError(memd.StreamEndBackfillFailed)
// ErrDCPStreamFilterEmpty occurs when all of the collections for a DCP stream are
// dropped.
ErrDCPStreamFilterEmpty = makeStreamEndStatusError(memd.StreamEndFilterEmpty)
// ErrStreamIDNotEnabled occurs when dcp operations are performed using a stream ID when stream IDs are not enabled.
ErrStreamIDNotEnabled = errors.New("stream IDs have not been enabled on this stream")
// ErrDCPStreamIDInvalid occurs when a dcp stream ID is invalid.
ErrDCPStreamIDInvalid = errors.New("stream ID invalid")
)
gocbcore-10.2.3/error_memd.go 0000664 0000000 0000000 00000026570 14417540156 0016063 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"log"
"github.com/couchbase/gocbcore/v10/memd"
)
var statusCodeErrorMap = make(map[memd.StatusCode]error)
func makeKvStatusError(code memd.StatusCode) error {
err := errors.New(code.String())
if statusCodeErrorMap[code] != nil {
log.Fatal("error handling setup failure")
}
statusCodeErrorMap[code] = err
return err
}
func getKvStatusCodeError(code memd.StatusCode) error {
if err := statusCodeErrorMap[code]; err != nil {
return err
}
return errors.New(code.String())
}
var (
// ErrMemdKeyNotFound occurs when an operation is performed on a key that does not exist.
ErrMemdKeyNotFound = makeKvStatusError(memd.StatusKeyNotFound)
// ErrMemdKeyExists occurs when an operation is performed on a key that could not be found.
ErrMemdKeyExists = makeKvStatusError(memd.StatusKeyExists)
// ErrMemdTooBig occurs when an operation attempts to store more data in a single document
// than the server is capable of storing (by default, this is a 20MB limit).
ErrMemdTooBig = makeKvStatusError(memd.StatusTooBig)
// ErrMemdInvalidArgs occurs when the server receives invalid arguments for an operation.
ErrMemdInvalidArgs = makeKvStatusError(memd.StatusInvalidArgs)
// ErrMemdNotStored occurs when the server fails to store a key.
ErrMemdNotStored = makeKvStatusError(memd.StatusNotStored)
// ErrMemdBadDelta occurs when an invalid delta value is specified to a counter operation.
ErrMemdBadDelta = makeKvStatusError(memd.StatusBadDelta)
// ErrMemdNotMyVBucket occurs when an operation is dispatched to a server which is
// non-authoritative for a specific vbucket.
ErrMemdNotMyVBucket = makeKvStatusError(memd.StatusNotMyVBucket)
// ErrMemdNoBucket occurs when no bucket was selected on a connection.
ErrMemdNoBucket = makeKvStatusError(memd.StatusNoBucket)
// ErrMemdLocked occurs when a document is already locked.
ErrMemdLocked = makeKvStatusError(memd.StatusLocked)
// ErrMemdAuthStale occurs when authentication credentials have become invalidated.
ErrMemdAuthStale = makeKvStatusError(memd.StatusAuthStale)
// ErrMemdAuthError occurs when the authentication information provided was not valid.
ErrMemdAuthError = makeKvStatusError(memd.StatusAuthError)
// ErrMemdAuthContinue occurs in multi-step authentication when more authentication
// work needs to be performed in order to complete the authentication process.
ErrMemdAuthContinue = makeKvStatusError(memd.StatusAuthContinue)
// ErrMemdRangeError occurs when the range specified to the server is not valid.
ErrMemdRangeError = makeKvStatusError(memd.StatusRangeError)
// ErrMemdRollback occurs when a DCP stream fails to open due to a rollback having
// previously occurred since the last time the stream was opened.
ErrMemdRollback = makeKvStatusError(memd.StatusRollback)
// ErrMemdAccessError occurs when an access error occurs.
ErrMemdAccessError = makeKvStatusError(memd.StatusAccessError)
// ErrMemdNotInitialized is sent by servers which are still initializing, and are not
// yet ready to accept operations on behalf of a particular bucket.
ErrMemdNotInitialized = makeKvStatusError(memd.StatusNotInitialized)
// ErrMemdUnknownCommand occurs when an unknown operation is sent to a server.
ErrMemdUnknownCommand = makeKvStatusError(memd.StatusUnknownCommand)
// ErrMemdOutOfMemory occurs when the server cannot service a request due to memory
// limitations.
ErrMemdOutOfMemory = makeKvStatusError(memd.StatusOutOfMemory)
// ErrMemdNotSupported occurs when an operation is understood by the server, but that
// operation is not supported on this server (occurs for a variety of reasons).
ErrMemdNotSupported = makeKvStatusError(memd.StatusNotSupported)
// ErrMemdInternalError occurs when internal errors prevent the server from processing
// your request.
ErrMemdInternalError = makeKvStatusError(memd.StatusInternalError)
// ErrMemdBusy occurs when the server is too busy to process your request right away.
// Attempting the operation at a later time will likely succeed.
ErrMemdBusy = makeKvStatusError(memd.StatusBusy)
// ErrMemdTmpFail occurs when a temporary failure is preventing the server from
// processing your request.
ErrMemdTmpFail = makeKvStatusError(memd.StatusTmpFail)
// ErrMemdCollectionNotFound occurs when a Collection cannot be found.
ErrMemdCollectionNotFound = makeKvStatusError(memd.StatusCollectionUnknown)
// ErrMemdScopeNotFound occurs when a Scope cannot be found.
ErrMemdScopeNotFound = makeKvStatusError(memd.StatusScopeUnknown)
// ErrMemdDCPStreamIDInvalid occurs when a dcp stream ID is invalid.
ErrMemdDCPStreamIDInvalid = makeKvStatusError(memd.StatusDCPStreamIDInvalid)
// ErrMemdDurabilityInvalidLevel occurs when an invalid durability level was requested.
ErrMemdDurabilityInvalidLevel = makeKvStatusError(memd.StatusDurabilityInvalidLevel)
// ErrMemdDurabilityImpossible occurs when a request is performed with impossible
// durability level requirements.
ErrMemdDurabilityImpossible = makeKvStatusError(memd.StatusDurabilityImpossible)
// ErrMemdSyncWriteInProgess occurs when an attempt is made to write to a key that has
// a SyncWrite pending.
ErrMemdSyncWriteInProgess = makeKvStatusError(memd.StatusSyncWriteInProgress)
// ErrMemdSyncWriteAmbiguous occurs when an SyncWrite does not complete in the specified
// time and the result is ambiguous.
ErrMemdSyncWriteAmbiguous = makeKvStatusError(memd.StatusSyncWriteAmbiguous)
// ErrMemdSyncWriteReCommitInProgress occurs when an SyncWrite is being recommitted.
ErrMemdSyncWriteReCommitInProgress = makeKvStatusError(memd.StatusSyncWriteReCommitInProgress)
// ErrMemdSubDocPathNotFound occurs when a sub-document operation targets a path
// which does not exist in the specifie document.
ErrMemdSubDocPathNotFound = makeKvStatusError(memd.StatusSubDocPathNotFound)
// ErrMemdSubDocPathMismatch occurs when a sub-document operation specifies a path
// which does not match the document structure (field access on an array).
ErrMemdSubDocPathMismatch = makeKvStatusError(memd.StatusSubDocPathMismatch)
// ErrMemdSubDocPathInvalid occurs when a sub-document path could not be parsed.
ErrMemdSubDocPathInvalid = makeKvStatusError(memd.StatusSubDocPathInvalid)
// ErrMemdSubDocPathTooBig occurs when a sub-document path is too big.
ErrMemdSubDocPathTooBig = makeKvStatusError(memd.StatusSubDocPathTooBig)
// ErrMemdSubDocDocTooDeep occurs when an operation would cause a document to be
// nested beyond the depth limits allowed by the sub-document specification.
ErrMemdSubDocDocTooDeep = makeKvStatusError(memd.StatusSubDocDocTooDeep)
// ErrMemdSubDocCantInsert occurs when a sub-document operation could not insert.
ErrMemdSubDocCantInsert = makeKvStatusError(memd.StatusSubDocCantInsert)
// ErrMemdSubDocNotJSON occurs when a sub-document operation is performed on a
// document which is not JSON.
ErrMemdSubDocNotJSON = makeKvStatusError(memd.StatusSubDocNotJSON)
// ErrMemdSubDocBadRange occurs when a sub-document operation is performed with
// a bad range.
ErrMemdSubDocBadRange = makeKvStatusError(memd.StatusSubDocBadRange)
// ErrMemdSubDocBadDelta occurs when a sub-document counter operation is performed
// and the specified delta is not valid.
ErrMemdSubDocBadDelta = makeKvStatusError(memd.StatusSubDocBadDelta)
// ErrMemdSubDocPathExists occurs when a sub-document operation expects a path not
// to exists, but the path was found in the document.
ErrMemdSubDocPathExists = makeKvStatusError(memd.StatusSubDocPathExists)
// ErrMemdSubDocValueTooDeep occurs when a sub-document operation specifies a value
// which is deeper than the depth limits of the sub-document specification.
ErrMemdSubDocValueTooDeep = makeKvStatusError(memd.StatusSubDocValueTooDeep)
// ErrMemdSubDocBadCombo occurs when a multi-operation sub-document operation is
// performed and operations within the package of ops conflict with each other.
ErrMemdSubDocBadCombo = makeKvStatusError(memd.StatusSubDocBadCombo)
// ErrMemdSubDocBadMulti occurs when a multi-operation sub-document operation is
// performed and operations within the package of ops conflict with each other.
ErrMemdSubDocBadMulti = makeKvStatusError(memd.StatusSubDocBadMulti)
// ErrMemdSubDocSuccessDeleted occurs when a multi-operation sub-document operation
// is performed on a soft-deleted document.
ErrMemdSubDocSuccessDeleted = makeKvStatusError(memd.StatusSubDocSuccessDeleted)
// ErrMemdSubDocXattrInvalidFlagCombo occurs when an invalid set of
// extended-attribute flags is passed to a sub-document operation.
ErrMemdSubDocXattrInvalidFlagCombo = makeKvStatusError(memd.StatusSubDocXattrInvalidFlagCombo)
// ErrMemdSubDocXattrInvalidKeyCombo occurs when an invalid set of key operations
// are specified for a extended-attribute sub-document operation.
ErrMemdSubDocXattrInvalidKeyCombo = makeKvStatusError(memd.StatusSubDocXattrInvalidKeyCombo)
// ErrMemdSubDocXattrUnknownMacro occurs when an invalid macro value is specified.
ErrMemdSubDocXattrUnknownMacro = makeKvStatusError(memd.StatusSubDocXattrUnknownMacro)
// ErrMemdSubDocXattrUnknownVAttr occurs when an invalid virtual attribute is specified.
ErrMemdSubDocXattrUnknownVAttr = makeKvStatusError(memd.StatusSubDocXattrUnknownVAttr)
// ErrMemdSubDocXattrCannotModifyVAttr occurs when a mutation is attempted upon
// a virtual attribute (which are immutable by definition).
ErrMemdSubDocXattrCannotModifyVAttr = makeKvStatusError(memd.StatusSubDocXattrCannotModifyVAttr)
// ErrMemdSubDocMultiPathFailureDeleted occurs when a Multi Path Failure occurs on
// a soft-deleted document.
ErrMemdSubDocMultiPathFailureDeleted = makeKvStatusError(memd.StatusSubDocMultiPathFailureDeleted)
// ErrMemdRateLimitedNetworkIngress occurs when the server rate limits due to network ingress.
ErrMemdRateLimitedNetworkIngress = makeKvStatusError(memd.StatusRateLimitedNetworkIngress)
// ErrMemdRateLimitedNetworkEgress occurs when the server rate limits due to network egress.
ErrMemdRateLimitedNetworkEgress = makeKvStatusError(memd.StatusRateLimitedNetworkEgress)
// ErrMemdRateLimitedMaxConnections occurs when the server rate limits due to the application reaching the maximum
// number of allowed connections.
ErrMemdRateLimitedMaxConnections = makeKvStatusError(memd.StatusRateLimitedMaxConnections)
// ErrMemdRateLimitedMaxCommands occurs when the server rate limits due to the application reaching the maximum
// number of allowed operations.
ErrMemdRateLimitedMaxCommands = makeKvStatusError(memd.StatusRateLimitedMaxCommands)
// ErrMemdRateLimitedScopeSizeLimitExceeded occurs when the server rate limits due to the application reaching the maximum
// data size allowed for the scope.
ErrMemdRateLimitedScopeSizeLimitExceeded = makeKvStatusError(memd.StatusRateLimitedScopeSizeLimitExceeded)
// ErrMemdRangeScanCancelled occurs during a range scan to indicate that the range scan was cancelled.
ErrMemdRangeScanCancelled = makeKvStatusError(memd.StatusRangeScanCancelled)
// ErrMemdRangeScanMore occurs during a range scan to indicate that a range scan has more results.
ErrMemdRangeScanMore = makeKvStatusError(memd.StatusRangeScanMore)
// ErrMemdRangeScanComplete occurs during a range scan to indicate that a range scan has completed.
ErrMemdRangeScanComplete = makeKvStatusError(memd.StatusRangeScanComplete)
// ErrMemdRangeScanVbUUIDNotEqual occurs during a range scan to indicate that a vb-uuid mismatch has occurred.
ErrMemdRangeScanVbUUIDNotEqual = makeKvStatusError(memd.StatusRangeScanVbUUIDNotEqual)
)
gocbcore-10.2.3/error_test.go 0000664 0000000 0000000 00000003440 14417540156 0016107 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"github.com/couchbase/gocbcore/v10/memd"
)
func (suite *StandardTestSuite) TestEnhancedErrors() {
agent, s := suite.GetAgentAndHarness()
var err1, err2 error
s.PushOp(agent.Get(GetOptions{
Key: []byte("keyThatWontExist"),
}, func(res *GetResult, err error) {
s.Wrap(func() {
err1 = err
})
}))
s.Wait(0)
s.PushOp(agent.Get(GetOptions{
Key: []byte("keyThatWontExist"),
}, func(res *GetResult, err error) {
s.Wrap(func() {
err2 = err
})
}))
s.Wait(0)
if err1 == err2 {
suite.T().Fatalf("Operation error results should never return equivalent values")
}
}
func (suite *StandardTestSuite) TestEnhancedErrorOp() {
suite.EnsureSupportsFeature(TestFeatureErrMap)
if !suite.IsMockServer() {
suite.T().Skipf("only supported when testing against mock server")
}
spec := suite.StartTest(TestNameExtendedError)
h := suite.GetHarness()
agent := spec.Agent
h.PushOp(agent.GetAndLock(GetAndLockOptions{
Key: []byte("testEnhancedErrs"),
LockTime: 10,
CollectionName: spec.Collection,
ScopeName: spec.Scope,
}, func(res *GetAndLockResult, err error) {
h.Wrap(func() {
typedErr, ok := err.(*KeyValueError)
if !ok {
h.Fatalf("error should be a KeyValueError: %v", err)
}
if typedErr.Context == "" {
h.Fatalf("error should have a context")
}
if typedErr.ErrorName == "" {
h.Fatalf("error should have a name")
}
if typedErr.ErrorDescription == "" {
h.Fatalf("error should have a description")
}
if typedErr.StatusCode != memd.StatusKeyNotFound {
h.Fatalf("status code should have been StatusKeyNotFound")
}
if !errors.Is(err, ErrDocumentNotFound) {
h.Fatalf("error cause should have been ErrDocumentNotFound")
}
})
}))
h.Wait(0)
suite.EndTest(spec)
}
gocbcore-10.2.3/error_transactions.go 0000664 0000000 0000000 00000021517 14417540156 0017645 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"fmt"
"strings"
)
var (
// ErrNoAttempt indicates no attempt was started before an operation was performed.
ErrNoAttempt = errors.New("attempt was not started")
// ErrOther indicates an non-specific error has occured.
ErrOther = errors.New("other error")
// ErrTransient indicates a transient error occured which may succeed at a later point in time.
ErrTransient = errors.New("transient error")
// ErrWriteWriteConflict indicates that another transaction conflicted with this one.
ErrWriteWriteConflict = errors.New("write write conflict")
// ErrHard indicates that an unrecoverable error occured.
ErrHard = errors.New("hard")
// ErrAmbiguous indicates that a failure occured but the outcome was not known.
ErrAmbiguous = errors.New("ambiguous error")
// ErrAtrFull indicates that the ATR record was too full to accept a new mutation.
ErrAtrFull = errors.New("atr full")
// ErrAttemptExpired indicates an attempt expired.
ErrAttemptExpired = errors.New("attempt expired")
// ErrAtrNotFound indicates that an expected ATR document was missing.
ErrAtrNotFound = errors.New("atr not found")
// ErrAtrEntryNotFound indicates that an expected ATR entry was missing.
ErrAtrEntryNotFound = errors.New("atr entry not found")
// ErrDocAlreadyInTransaction indicates that a document is already in a transaction.
ErrDocAlreadyInTransaction = errors.New("doc already in transaction")
// ErrIllegalState is used for when a transaction enters an illegal State.
ErrIllegalState = errors.New("illegal State")
// ErrTransactionAbortedExternally indicates the transaction was aborted externally.
ErrTransactionAbortedExternally = errors.New("transaction aborted externally")
// ErrPreviousOperationFailed indicates a previous operation in the transaction failed.
ErrPreviousOperationFailed = errors.New("previous operation failed")
// ErrForwardCompatibilityFailure indicates an operation failed due to involving a document in another transaction
// which contains features this transaction does not support.
ErrForwardCompatibilityFailure = errors.New("forward compatibility error")
)
type classifiedError struct {
Source error
Class TransactionErrorClass
}
func (ce classifiedError) Wrap(errType error) *classifiedError {
return &classifiedError{
Source: &basicRetypedError{
ErrType: errType,
Source: ce.Source,
},
Class: ce.Class,
}
}
// TransactionOperationFailedError is used when a transaction operation fails.
// Internal: This should never be used and is not supported.
type TransactionOperationFailedError struct {
shouldNotRetry bool
shouldNotRollback bool
errorCause error
shouldRaise TransactionErrorReason
errorClass TransactionErrorClass
}
// MarshalJSON will marshal this error for the wire.
func (tfe TransactionOperationFailedError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
Retry bool `json:"retry"`
Rollback bool `json:"rollback"`
Raise string `json:"raise"`
Cause json.RawMessage `json:"cause"`
}{
Retry: !tfe.shouldNotRetry,
Rollback: !tfe.shouldNotRollback,
Raise: tfe.shouldRaise.String(),
Cause: marshalErrorToJSON(tfe.errorCause),
})
}
func (tfe TransactionOperationFailedError) Error() string {
errStr := "transaction operation failed"
errStr += " | " + fmt.Sprintf(
"shouldRetry:%v, shouldRollback:%v, shouldRaise:%d, class:%d",
!tfe.shouldNotRetry,
!tfe.shouldNotRollback,
tfe.shouldRaise,
tfe.errorClass)
if tfe.errorCause != nil {
errStr += " | " + tfe.errorCause.Error()
}
return errStr
}
// Retry signals whether a new attempt should be made at rollback.
func (tfe TransactionOperationFailedError) Retry() bool {
return !tfe.shouldNotRetry
}
// Rollback signals whether the attempt should be auto-rolled back.
func (tfe TransactionOperationFailedError) Rollback() bool {
return !tfe.shouldNotRollback
}
// ToRaise signals which error type should be raised to the application.
func (tfe TransactionOperationFailedError) ToRaise() TransactionErrorReason {
return tfe.shouldRaise
}
// ErrorClass returns the class of error which caused this error.
func (tfe TransactionOperationFailedError) ErrorClass() TransactionErrorClass {
return tfe.errorClass
}
// InternalUnwrap returns the underlying error for this error.
func (tfe TransactionOperationFailedError) InternalUnwrap() error {
return tfe.errorCause
}
type aggregateError []error
func (agge aggregateError) MarshalJSON() ([]byte, error) {
suberrs := make([]json.RawMessage, len(agge))
for i, err := range agge {
suberrs[i] = marshalErrorToJSON(err)
}
return json.Marshal(suberrs)
}
func (agge aggregateError) Error() string {
errStrs := []string{}
for _, err := range agge {
errStrs = append(errStrs, err.Error())
}
return "[" + strings.Join(errStrs, ", ") + "]"
}
func (agge aggregateError) Is(err error) bool {
for _, aerr := range agge {
if errors.Is(aerr, err) {
return true
}
}
return false
}
type writeWriteConflictError struct {
BucketName string
ScopeName string
CollectionName string
DocumentKey []byte
Source error
}
func (wwce writeWriteConflictError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
Msg string `json:"msg"`
Cause json.RawMessage `json:"cause"`
BucketName string `json:"bucket"`
ScopeName string `json:"scope"`
CollectionName string `json:"collection"`
DocumentKey string `json:"document_key"`
}{
Msg: "write write conflict",
Cause: marshalErrorToJSON(wwce.Source),
BucketName: wwce.BucketName,
ScopeName: wwce.ScopeName,
CollectionName: wwce.CollectionName,
DocumentKey: string(wwce.DocumentKey),
})
}
func (wwce writeWriteConflictError) Error() string {
errStr := "write write conflict"
errStr += " | " + fmt.Sprintf(
"bucket:%s, scope:%s, collection:%s, key:%s",
wwce.BucketName,
wwce.ScopeName,
wwce.CollectionName,
wwce.DocumentKey)
if wwce.Source != nil {
errStr += " | " + wwce.Source.Error()
}
return errStr
}
func (wwce writeWriteConflictError) Is(err error) bool {
if err == ErrWriteWriteConflict {
return true
}
return errors.Is(wwce.Source, err)
}
func (wwce writeWriteConflictError) Unwrap() error {
return wwce.Source
}
type basicRetypedError struct {
ErrType error
Source error
}
func (bre basicRetypedError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
Msg string `json:"msg"`
Cause json.RawMessage `json:"cause"`
}{
Msg: bre.ErrType.Error(),
Cause: marshalErrorToJSON(bre.Source),
})
}
func (bre basicRetypedError) Error() string {
errStr := bre.ErrType.Error()
if bre.Source != nil {
errStr += " | " + bre.Source.Error()
}
return errStr
}
func (bre basicRetypedError) Is(err error) bool {
if errors.Is(bre.ErrType, err) {
return true
}
return errors.Is(bre.Source, err)
}
func (bre basicRetypedError) Unwrap() error {
return bre.Source
}
type forwardCompatError struct {
BucketName string
ScopeName string
CollectionName string
DocumentKey []byte
}
func (fce forwardCompatError) Error() string {
errStr := ErrForwardCompatibilityFailure.Error()
errStr += " | " + fmt.Sprintf(
"bucket:%s, scope:%s, collection:%s, key:%s",
fce.BucketName,
fce.ScopeName,
fce.CollectionName,
fce.DocumentKey)
return errStr
}
func (fce forwardCompatError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
BucketName string `json:"bucket,omitempty"`
ScopeName string `json:"scope,omitempty"`
CollectionName string `json:"collection,omitempty"`
DocumentKey string `json:"document_key,omitempty"`
Message string `json:"msg"`
}{
BucketName: fce.BucketName,
ScopeName: fce.ScopeName,
CollectionName: fce.CollectionName,
DocumentKey: string(fce.DocumentKey),
Message: ErrForwardCompatibilityFailure.Error(),
})
}
func (fce forwardCompatError) Unwrap() error {
return ErrForwardCompatibilityFailure
}
func marshalErrorToJSON(err error) json.RawMessage {
if marshaler, ok := err.(json.Marshaler); ok {
if data, err := marshaler.MarshalJSON(); err == nil {
return data
}
}
data, err := json.Marshal(err.Error())
if err != nil {
logWarnf("Failed to marshal error: %v", err)
}
return data
}
gocbcore-10.2.3/error_transactions_test.go 0000664 0000000 0000000 00000006210 14417540156 0020675 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"github.com/couchbase/gocbcore/v10/memd"
)
func (suite *UnitTestSuite) TestAggregateErrorMarshals() {
terr := &aggregateError{
errors.New("some-error"),
&TransactionOperationFailedError{
shouldNotRetry: true,
shouldNotRollback: true,
errorCause: errors.New("some-cause"),
shouldRaise: TransactionErrorReasonTransactionExpired,
errorClass: TransactionErrorClassFailCasMismatch,
},
}
bytes, err := json.Marshal(terr)
suite.Require().Nil(err, "marshal failed")
suite.Require().Equal([]byte(`["some-error",{"retry":false,"rollback":false,"raise":"expired","cause":"some-cause"}]`), bytes)
}
func (suite *UnitTestSuite) TestGocbcoreErrorMarshals() {
terr := &TransactionOperationFailedError{
shouldNotRetry: true,
shouldNotRollback: true,
errorCause: KeyValueError{
InnerError: ErrCasMismatch,
StatusCode: memd.StatusAccessError,
DocumentKey: "key",
BucketName: "bucket",
ScopeName: "scope",
CollectionName: "collection",
CollectionID: 19,
ErrorName: "",
ErrorDescription: "",
Opaque: 4019,
Context: "",
Ref: "",
RetryReasons: nil,
RetryAttempts: 1,
LastDispatchedTo: "127.0.0.1:11210",
LastDispatchedFrom: "127.0.0.1:79654",
LastConnectionID: "",
},
shouldRaise: TransactionErrorReasonTransactionExpired,
errorClass: TransactionErrorClassFailCasMismatch,
}
bytes, err := json.Marshal(terr)
suite.Require().Nil(err, "marshal failed")
suite.Require().Equal([]byte(`{"retry":false,"rollback":false,"raise":"expired","cause":{"msg":"cas mismatch","status_code":36,"document_key":"key","bucket":"bucket","scope":"scope","collection":"collection","collection_id":19,"opaque":4019,"retry_attempts":1,"last_dispatched_to":"127.0.0.1:11210","last_dispatched_from":"127.0.0.1:79654"}}`), bytes)
}
func (suite *UnitTestSuite) TestForwardCompatError() {
terr := &forwardCompatError{
DocumentKey: []byte("key"),
BucketName: "bucket",
ScopeName: "scope",
CollectionName: "collection",
}
suite.Assert().ErrorIs(terr, ErrForwardCompatibilityFailure)
suite.Assert().Equal(`forward compatibility error | bucket:bucket, scope:scope, collection:collection, key:key`, terr.Error())
bytes, err := json.Marshal(terr)
suite.Require().Nil(err, "marshal failed")
suite.Assert().Equal(`{"bucket":"bucket","scope":"scope","collection":"collection","document_key":"key","msg":"forward compatibility error"}`, string(bytes))
}
gocbcore-10.2.3/errors_internal.go 0000664 0000000 0000000 00000072307 14417540156 0017137 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
type wrappedError struct {
Message string
InnerError error
}
func (e wrappedError) Error() string {
return fmt.Sprintf("%s: %s", e.Message, e.InnerError.Error())
}
func (e wrappedError) Unwrap() error {
return e.InnerError
}
func wrapError(err error, message string) error {
return wrappedError{
Message: message,
InnerError: err,
}
}
// SubDocumentError provides additional contextual information to
// sub-document specific errors. InnerError is always a KeyValueError.
type SubDocumentError struct {
InnerError error
Index int
}
// Error returns the string representation of this error.
func (err SubDocumentError) Error() string {
return fmt.Sprintf("sub-document error at index %d: %s",
err.Index,
err.InnerError.Error())
}
// Unwrap returns the underlying error for the operation failing.
func (err SubDocumentError) Unwrap() error {
return err.InnerError
}
func serializeError(err error) string {
errBytes, serErr := json.Marshal(err)
if serErr != nil {
logErrorf("failed to serialize error to json: %s", serErr.Error())
}
return string(errBytes)
}
// KeyValueError wraps key-value errors that occur within the SDK.
type KeyValueError struct {
InnerError error
StatusCode memd.StatusCode
DocumentKey string
BucketName string
ScopeName string
CollectionName string
CollectionID uint32
ErrorName string
ErrorDescription string
Opaque uint32
Context string
Ref string
RetryReasons []RetryReason
RetryAttempts uint32
LastDispatchedTo string
LastDispatchedFrom string
LastConnectionID string
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
// MarshalJSON implements the Marshaler interface.
func (e KeyValueError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
InnerError string `json:"msg,omitempty"`
StatusCode memd.StatusCode `json:"status_code,omitempty"`
DocumentKey string `json:"document_key,omitempty"`
BucketName string `json:"bucket,omitempty"`
ScopeName string `json:"scope,omitempty"`
CollectionName string `json:"collection,omitempty"`
CollectionID uint32 `json:"collection_id,omitempty"`
ErrorName string `json:"error_name,omitempty"`
ErrorDescription string `json:"error_description,omitempty"`
Opaque uint32 `json:"opaque,omitempty"`
Context string `json:"context,omitempty"`
Ref string `json:"ref,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
LastDispatchedTo string `json:"last_dispatched_to,omitempty"`
LastDispatchedFrom string `json:"last_dispatched_from,omitempty"`
LastConnectionID string `json:"last_connection_id,omitempty"`
}{
InnerError: e.InnerError.Error(),
StatusCode: e.StatusCode,
DocumentKey: e.DocumentKey,
BucketName: e.BucketName,
ScopeName: e.ScopeName,
CollectionName: e.CollectionName,
CollectionID: e.CollectionID,
ErrorName: e.ErrorName,
ErrorDescription: e.ErrorDescription,
Opaque: e.Opaque,
Context: e.Context,
Ref: e.Ref,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
LastDispatchedTo: e.LastDispatchedTo,
LastDispatchedFrom: e.LastDispatchedFrom,
LastConnectionID: e.LastConnectionID,
})
}
// Error returns the string representation of this error.
func (e KeyValueError) Error() string {
errBytes, serErr := json.Marshal(struct {
InnerError error `json:"-"`
StatusCode memd.StatusCode `json:"status_code,omitempty"`
DocumentKey string `json:"document_key,omitempty"`
BucketName string `json:"bucket,omitempty"`
ScopeName string `json:"scope,omitempty"`
CollectionName string `json:"collection,omitempty"`
CollectionID uint32 `json:"collection_id,omitempty"`
ErrorName string `json:"error_name,omitempty"`
ErrorDescription string `json:"error_description,omitempty"`
Opaque uint32 `json:"opaque,omitempty"`
Context string `json:"context,omitempty"`
Ref string `json:"ref,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
LastDispatchedTo string `json:"last_dispatched_to,omitempty"`
LastDispatchedFrom string `json:"last_dispatched_from,omitempty"`
LastConnectionID string `json:"last_connection_id,omitempty"`
}{
InnerError: e.InnerError,
StatusCode: e.StatusCode,
DocumentKey: e.DocumentKey,
BucketName: e.BucketName,
ScopeName: e.ScopeName,
CollectionName: e.CollectionName,
CollectionID: e.CollectionID,
ErrorName: e.ErrorName,
ErrorDescription: e.ErrorDescription,
Opaque: e.Opaque,
Context: e.Context,
Ref: e.Ref,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
LastDispatchedTo: e.LastDispatchedTo,
LastDispatchedFrom: e.LastDispatchedFrom,
LastConnectionID: e.LastConnectionID,
})
if serErr != nil {
logErrorf("failed to serialize error to json: %s", serErr.Error())
}
return e.InnerError.Error() + " | " + string(errBytes)
}
// Unwrap returns the underlying reason for the error
func (e KeyValueError) Unwrap() error {
return e.InnerError
}
// ViewQueryErrorDesc represents specific view error data.
type ViewQueryErrorDesc struct {
SourceNode string
Message string
}
// ViewError represents an error returned from a view query.
type ViewError struct {
InnerError error
DesignDocumentName string
ViewName string
Errors []ViewQueryErrorDesc
Endpoint string
RetryReasons []RetryReason
RetryAttempts uint32
// Uncommitted: This API may change in the future.
ErrorText string
// Uncommitted: This API may change in the future.
HTTPResponseCode int
}
// MarshalJSON implements the Marshaler interface.
func (e ViewError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
InnerError string `json:"msg,omitempty"`
DesignDocumentName string `json:"design_document_name,omitempty"`
ViewName string `json:"view_name,omitempty"`
Errors []ViewQueryErrorDesc `json:"errors,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
HTTPResponseCode int `json:"status_code,omitempty"`
}{
InnerError: e.InnerError.Error(),
DesignDocumentName: e.DesignDocumentName,
ViewName: e.ViewName,
Errors: e.Errors,
Endpoint: e.Endpoint,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
HTTPResponseCode: e.HTTPResponseCode,
})
}
// Error returns the string representation of this error.
func (e ViewError) Error() string {
errBytes, serErr := json.Marshal(struct {
InnerError error `json:"-"`
DesignDocumentName string `json:"design_document_name,omitempty"`
ViewName string `json:"view_name,omitempty"`
Errors []ViewQueryErrorDesc `json:"errors,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
ErrorText string `json:"error_text,omitempty"`
HTTPResponseCode int `json:"status_code,omitempty"`
}{
InnerError: e.InnerError,
DesignDocumentName: e.DesignDocumentName,
ViewName: e.ViewName,
Errors: e.Errors,
Endpoint: e.Endpoint,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
ErrorText: e.ErrorText,
HTTPResponseCode: e.HTTPResponseCode,
})
if serErr != nil {
logErrorf("failed to serialize error to json: %s", serErr.Error())
}
return e.InnerError.Error() + " | " + string(errBytes)
}
// Unwrap returns the underlying reason for the error
func (e ViewError) Unwrap() error {
return e.InnerError
}
// N1QLErrorDesc represents specific n1ql error data.
type N1QLErrorDesc struct {
Code uint32
Message string
Retry bool
Reason map[string]interface{}
}
// MarshalJSON implements the Marshaler interface.
func (e N1QLErrorDesc) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
Code uint32 `json:"code"`
Message string `json:"message"`
Retry bool `json:"retry,omitempty"`
Reason map[string]interface{} `json:"reason,omitempty"`
}{
Code: e.Code,
Message: e.Message,
Retry: e.Retry,
Reason: e.Reason,
})
}
// N1QLError represents an error returned from a n1ql query.
type N1QLError struct {
InnerError error
Statement string
ClientContextID string
Errors []N1QLErrorDesc
Endpoint string
RetryReasons []RetryReason
RetryAttempts uint32
// Uncommitted: This API may change in the future.
ErrorText string
// Uncommitted: This API may change in the future.
HTTPResponseCode int
}
// MarshalJSON implements the Marshaler interface.
func (e N1QLError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
InnerError string `json:"msg,omitempty"`
Statement string `json:"statement,omitempty"`
ClientContextID string `json:"client_context_id,omitempty"`
Errors []N1QLErrorDesc `json:"errors,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
HTTPResponseCode int `json:"status_code,omitempty"`
}{
InnerError: e.InnerError.Error(),
Statement: e.Statement,
ClientContextID: e.ClientContextID,
Errors: e.Errors,
Endpoint: e.Endpoint,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
HTTPResponseCode: e.HTTPResponseCode,
})
}
// Error returns the string representation of this error.
func (e N1QLError) Error() string {
errBytes, serErr := json.Marshal(struct {
InnerError error `json:"-"`
Statement string `json:"statement,omitempty"`
ClientContextID string `json:"client_context_id,omitempty"`
Errors []N1QLErrorDesc `json:"errors,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
ErrorText string `json:"error_text,omitempty"`
HTTPResponseCode int `json:"status_code,omitempty"`
}{
InnerError: e.InnerError,
Statement: e.Statement,
ClientContextID: e.ClientContextID,
Errors: e.Errors,
Endpoint: e.Endpoint,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
ErrorText: e.ErrorText,
HTTPResponseCode: e.HTTPResponseCode,
})
if serErr != nil {
logErrorf("failed to serialize error to json: %s", serErr.Error())
}
return e.InnerError.Error() + " | " + string(errBytes)
}
// Unwrap returns the underlying reason for the error
func (e N1QLError) Unwrap() error {
return e.InnerError
}
// AnalyticsErrorDesc represents specific analytics error data.
type AnalyticsErrorDesc struct {
Code uint32
Message string
}
// AnalyticsError represents an error returned from an analytics query.
type AnalyticsError struct {
InnerError error
Statement string
ClientContextID string
Errors []AnalyticsErrorDesc
Endpoint string
RetryReasons []RetryReason
RetryAttempts uint32
// Uncommitted: This API may change in the future.
ErrorText string
// Uncommitted: This API may change in the future.
HTTPResponseCode int
}
// MarshalJSON implements the Marshaler interface.
func (e AnalyticsError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
InnerError string `json:"msg,omitempty"`
Statement string `json:"statement,omitempty"`
ClientContextID string `json:"client_context_id,omitempty"`
Errors []AnalyticsErrorDesc `json:"errors,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
HTTPResponseCode int `json:"status_code,omitempty"`
}{
InnerError: e.InnerError.Error(),
Statement: e.Statement,
ClientContextID: e.ClientContextID,
Errors: e.Errors,
Endpoint: e.Endpoint,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
HTTPResponseCode: e.HTTPResponseCode,
})
}
// Error returns the string representation of this error.
func (e AnalyticsError) Error() string {
errBytes, serErr := json.Marshal(struct {
InnerError error `json:"-"`
Statement string `json:"statement,omitempty"`
ClientContextID string `json:"client_context_id,omitempty"`
Errors []AnalyticsErrorDesc `json:"errors,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
ErrorText string `json:"error_text,omitempty"`
HTTPResponseCode int `json:"status_code,omitempty"`
}{
InnerError: e.InnerError,
Statement: e.Statement,
ClientContextID: e.ClientContextID,
Errors: e.Errors,
Endpoint: e.Endpoint,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
ErrorText: e.ErrorText,
HTTPResponseCode: e.HTTPResponseCode,
})
if serErr != nil {
logErrorf("failed to serialize error to json: %s", serErr.Error())
}
return e.InnerError.Error() + " | " + string(errBytes)
}
// Unwrap returns the underlying reason for the error
func (e AnalyticsError) Unwrap() error {
return e.InnerError
}
// SearchError represents an error returned from a search query.
type SearchError struct {
InnerError error
IndexName string
Query interface{}
ErrorText string
HTTPResponseCode int
Endpoint string
RetryReasons []RetryReason
RetryAttempts uint32
}
// MarshalJSON implements the Marshaler interface.
func (e SearchError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
InnerError string `json:"msg,omitempty"`
IndexName string `json:"index_name,omitempty"`
Query interface{} `json:"query,omitempty"`
ErrorText string `json:"error_text"`
HTTPResponseCode int `json:"status_code,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
}{
InnerError: e.InnerError.Error(),
IndexName: e.IndexName,
Query: e.Query,
ErrorText: e.ErrorText,
HTTPResponseCode: e.HTTPResponseCode,
Endpoint: e.Endpoint,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
})
}
// Error returns the string representation of this error.
func (e SearchError) Error() string {
errBytes, serErr := json.Marshal(struct {
InnerError error `json:"-"`
IndexName string `json:"index_name,omitempty"`
Query interface{} `json:"query,omitempty"`
ErrorText string `json:"error_text"`
HTTPResponseCode int `json:"status_code,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
}{
InnerError: e.InnerError,
IndexName: e.IndexName,
Query: e.Query,
ErrorText: e.ErrorText,
HTTPResponseCode: e.HTTPResponseCode,
Endpoint: e.Endpoint,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
})
if serErr != nil {
logErrorf("failed to serialize error to json: %s", serErr.Error())
}
return e.InnerError.Error() + " | " + string(errBytes)
}
// Unwrap returns the underlying reason for the error
func (e SearchError) Unwrap() error {
return e.InnerError
}
// HTTPError represents an error returned from an HTTP request.
type HTTPError struct {
InnerError error
UniqueID string
Endpoint string
RetryReasons []RetryReason
RetryAttempts uint32
}
// MarshalJSON implements the Marshaler interface.
func (e HTTPError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
InnerError string `json:"msg,omitempty"`
UniqueID string `json:"unique_id,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
}{
InnerError: e.InnerError.Error(),
UniqueID: e.UniqueID,
Endpoint: e.Endpoint,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
})
}
// Error returns the string representation of this error.
func (e HTTPError) Error() string {
errBytes, serErr := json.Marshal(struct {
InnerError error `json:"-"`
UniqueID string `json:"unique_id,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
RetryReasons []RetryReason `json:"retry_reasons,omitempty"`
RetryAttempts uint32 `json:"retry_attempts,omitempty"`
}{
InnerError: e.InnerError,
UniqueID: e.UniqueID,
Endpoint: e.Endpoint,
RetryReasons: e.RetryReasons,
RetryAttempts: e.RetryAttempts,
})
if serErr != nil {
logErrorf("failed to serialize error to json: %s", serErr.Error())
}
return e.InnerError.Error() + " | " + string(errBytes)
}
// Unwrap returns the underlying reason for the error
func (e HTTPError) Unwrap() error {
return e.InnerError
}
// TimeoutError wraps timeout errors that occur within the SDK.
type TimeoutError struct {
InnerError error
OperationID string
Opaque string
TimeObserved time.Duration
RetryReasons []RetryReason
RetryAttempts uint32
LastDispatchedTo string
LastDispatchedFrom string
LastConnectionID string
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnits *ResourceUnitResult
}
}
func makeTimeoutError(start time.Time, op string, innerErr error, req *memdQRequest) *TimeoutError {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
err := &TimeoutError{
InnerError: innerErr,
OperationID: op,
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
}
err.Internal.ResourceUnits = req.ResourceUnits()
return err
}
type timeoutError struct {
InnerError error `json:"-,omitempty"`
OperationID string `json:"s,omitempty"`
Opaque string `json:"i,omitempty"`
TimeObserved uint64 `json:"t,omitempty"`
RetryReasons []RetryReason `json:"rr,omitempty"`
RetryAttempts uint32 `json:"ra,omitempty"`
LastDispatchedTo string `json:"r,omitempty"`
LastDispatchedFrom string `json:"l,omitempty"`
LastConnectionID string `json:"c,omitempty"`
}
// MarshalJSON implements the Marshaler interface.
func (err *TimeoutError) MarshalJSON() ([]byte, error) {
toMarshal := timeoutError{
InnerError: err.InnerError,
OperationID: err.OperationID,
Opaque: err.Opaque,
TimeObserved: uint64(err.TimeObserved / time.Microsecond),
RetryReasons: err.RetryReasons,
RetryAttempts: err.RetryAttempts,
LastDispatchedTo: err.LastDispatchedTo,
LastDispatchedFrom: err.LastDispatchedFrom,
LastConnectionID: err.LastConnectionID,
}
return json.Marshal(toMarshal)
}
// UnmarshalJSON implements the Unmarshaler interface.
func (err *TimeoutError) UnmarshalJSON(data []byte) error {
var tErr timeoutError
if jErr := json.Unmarshal(data, &tErr); jErr != nil {
return jErr
}
duration := time.Duration(tErr.TimeObserved) * time.Microsecond
err.InnerError = tErr.InnerError
err.OperationID = tErr.OperationID
err.Opaque = tErr.Opaque
err.TimeObserved = duration
err.RetryReasons = tErr.RetryReasons
err.RetryAttempts = tErr.RetryAttempts
err.LastDispatchedTo = tErr.LastDispatchedTo
err.LastDispatchedFrom = tErr.LastDispatchedFrom
err.LastConnectionID = tErr.LastConnectionID
return nil
}
func (err TimeoutError) Error() string {
return err.InnerError.Error() + " | " + serializeError(err)
}
// Unwrap returns the underlying reason for the error
func (err TimeoutError) Unwrap() error {
return err.InnerError
}
type DCPRollbackError struct {
InnerError error
SeqNo SeqNo
}
// MarshalJSON implements the Marshaler interface.
func (e DCPRollbackError) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
InnerError string `json:"msg,omitempty"`
SeqNo uint64 `json:"seq_no,omitempty"`
}{
InnerError: e.InnerError.Error(),
SeqNo: uint64(e.SeqNo),
})
}
// Error returns the string representation of this error.
func (e DCPRollbackError) Error() string {
errBytes, serErr := json.Marshal(struct {
InnerError error `json:"-"`
SeqNo uint64 `json:"seq_no,omitempty"`
}{
InnerError: e.InnerError,
SeqNo: uint64(e.SeqNo),
})
if serErr != nil {
logErrorf("failed to serialize error to json: %s", serErr.Error())
}
return e.InnerError.Error() + " | " + string(errBytes)
}
// Unwrap returns the underlying reason for the error
func (err DCPRollbackError) Unwrap() error {
return err.InnerError
}
// ncError is a wrapper error that provides no additional context to one of the
// publicly exposed error types. This is to force people to correctly use the
// error handling behaviours to check the error, rather than direct compares.
type ncError struct {
InnerError error
}
func (err ncError) Error() string {
return err.InnerError.Error()
}
func (err ncError) Unwrap() error {
return err.InnerError
}
func isErrorStatus(err error, code memd.StatusCode) bool {
var kvErr *KeyValueError
if errors.As(err, &kvErr) {
return kvErr.StatusCode == code
}
return false
}
var (
// errCircuitBreakerOpen is passed around internally to signal that an
// operation was cancelled due to the circuit breaker being open.
errCircuitBreakerOpen = errors.New("circuit breaker open")
errNoCCCPHosts = errors.New("no cccp hosts available")
)
// This list contains protected versions of all the errors we throw
// to ensure no users inadvertently rely on direct comparisons.
// nolint: deadcode,varcheck
var (
errTimeout = ncError{ErrTimeout}
errRequestCanceled = ncError{ErrRequestCanceled}
errInvalidArgument = ncError{ErrInvalidArgument}
errServiceNotAvailable = ncError{ErrServiceNotAvailable}
errInternalServerFailure = ncError{ErrInternalServerFailure}
errAuthenticationFailure = ncError{ErrAuthenticationFailure}
errTemporaryFailure = ncError{ErrTemporaryFailure}
errParsingFailure = ncError{ErrParsingFailure}
errCasMismatch = ncError{ErrCasMismatch}
errBucketNotFound = ncError{ErrBucketNotFound}
errCollectionNotFound = ncError{ErrCollectionNotFound}
errEncodingFailure = ncError{ErrEncodingFailure}
errDecodingFailure = ncError{ErrDecodingFailure}
errUnsupportedOperation = ncError{ErrUnsupportedOperation}
errAmbiguousTimeout = ncError{ErrAmbiguousTimeout}
errUnambiguousTimeout = ncError{ErrUnambiguousTimeout}
errFeatureNotAvailable = ncError{ErrFeatureNotAvailable}
errScopeNotFound = ncError{ErrScopeNotFound}
errIndexNotFound = ncError{ErrIndexNotFound}
errIndexExists = ncError{ErrIndexExists}
errGCCCPInUse = ncError{ErrGCCCPInUse}
errNotMyVBucket = ncError{ErrNotMyVBucket}
errDMLFailure = ncError{ErrDMLFailure}
errMemdClientClosed = ncError{ErrMemdClientClosed}
errRequestAlreadyDispatched = ncError{ErrRequestAlreadyDispatched}
errDocumentNotFound = ncError{ErrDocumentNotFound}
errDocumentUnretrievable = ncError{ErrDocumentUnretrievable}
errDocumentLocked = ncError{ErrDocumentLocked}
errValueTooLarge = ncError{ErrValueTooLarge}
errDocumentExists = ncError{ErrDocumentExists}
errNotStored = ncError{ErrNotStored}
errValueNotJSON = ncError{ErrValueNotJSON}
errDurabilityLevelNotAvailable = ncError{ErrDurabilityLevelNotAvailable}
errDurabilityImpossible = ncError{ErrDurabilityImpossible}
errDurabilityAmbiguous = ncError{ErrDurabilityAmbiguous}
errDurableWriteInProgress = ncError{ErrDurableWriteInProgress}
errDurableWriteReCommitInProgress = ncError{ErrDurableWriteReCommitInProgress}
errMutationLost = ncError{ErrMutationLost}
errPathNotFound = ncError{ErrPathNotFound}
errPathMismatch = ncError{ErrPathMismatch}
errPathInvalid = ncError{ErrPathInvalid}
errPathTooBig = ncError{ErrPathTooBig}
errPathTooDeep = ncError{ErrPathTooDeep}
errValueTooDeep = ncError{ErrValueTooDeep}
errValueInvalid = ncError{ErrValueInvalid}
errDocumentNotJSON = ncError{ErrDocumentNotJSON}
errNumberTooBig = ncError{ErrNumberTooBig}
errDeltaInvalid = ncError{ErrDeltaInvalid}
errPathExists = ncError{ErrPathExists}
errXattrUnknownMacro = ncError{ErrXattrUnknownMacro}
errXattrInvalidFlagCombo = ncError{ErrXattrInvalidFlagCombo}
errXattrInvalidKeyCombo = ncError{ErrXattrInvalidKeyCombo}
errXattrUnknownVirtualAttribute = ncError{ErrXattrUnknownVirtualAttribute}
errXattrCannotModifyVirtualAttribute = ncError{ErrXattrCannotModifyVirtualAttribute}
errXattrInvalidOrder = ncError{ErrXattrInvalidOrder}
errPlanningFailure = ncError{ErrPlanningFailure}
errIndexFailure = ncError{ErrIndexFailure}
errPreparedStatementFailure = ncError{ErrPreparedStatementFailure}
errCompilationFailure = ncError{ErrCompilationFailure}
errJobQueueFull = ncError{ErrJobQueueFull}
errDatasetNotFound = ncError{ErrDatasetNotFound}
errDataverseNotFound = ncError{ErrDataverseNotFound}
errDatasetExists = ncError{ErrDatasetExists}
errDataverseExists = ncError{ErrDataverseExists}
errLinkNotFound = ncError{ErrLinkNotFound}
errViewNotFound = ncError{ErrViewNotFound}
errDesignDocumentNotFound = ncError{ErrDesignDocumentNotFound}
errNoSupportedMechanisms = ncError{ErrNoSupportedMechanisms}
errBadHosts = ncError{ErrBadHosts}
errProtocol = ncError{ErrProtocol}
errNoReplicas = ncError{ErrNoReplicas}
errCliInternalError = ncError{ErrCliInternalError}
errInvalidCredentials = ncError{ErrInvalidCredentials}
errInvalidServer = ncError{ErrInvalidServer}
errInvalidVBucket = ncError{ErrInvalidVBucket}
errInvalidReplica = ncError{ErrInvalidReplica}
errInvalidService = ncError{ErrInvalidService}
errInvalidCertificate = ncError{ErrInvalidCertificate}
errCollectionsUnsupported = ncError{ErrCollectionsUnsupported}
errBucketAlreadySelected = ncError{ErrBucketAlreadySelected}
errShutdown = ncError{ErrShutdown}
errOverload = ncError{ErrOverload}
errStreamIDNotEnabled = ncError{ErrStreamIDNotEnabled}
errDCPStreamIDInvalid = ncError{ErrDCPStreamIDInvalid}
errForcedReconnect = ncError{ErrForcedReconnect}
errRateLimitedFailure = ncError{ErrRateLimitedFailure}
errQuotaLimitedFailure = ncError{ErrQuotaLimitedFailure}
errRangeScanCancelled = ncError{ErrRangeScanCancelled}
errRangeScanMore = ncError{ErrRangeScanMore}
errRangeScanComplete = ncError{ErrRangeScanComplete}
errRangeScanVbUUIDNotEqual = ncError{ErrRangeScanVbUUIDNotEqual}
)
gocbcore-10.2.3/go.mod 0000664 0000000 0000000 00000000357 14417540156 0014502 0 ustar 00root root 0000000 0000000 module github.com/couchbase/gocbcore/v10
require (
github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1
github.com/golang/snappy v0.0.4
github.com/google/uuid v1.3.0
github.com/stretchr/testify v1.8.2
)
go 1.13
gocbcore-10.2.3/go.sum 0000664 0000000 0000000 00000004205 14417540156 0014523 0 ustar 00root root 0000000 0000000 github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1 h1:H7OK4q4WsDxqNIB/Ba8BQBXBHFilZnyItHrLr3qmsKA=
github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gocbcore-10.2.3/hlcs.go 0000664 0000000 0000000 00000005027 14417540156 0014653 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"errors"
"fmt"
"strconv"
)
// From Java impl:
// ${Mutation.CAS} is written by kvengine with 'macroToString(htonll(info.cas))'. Discussed this with KV team and,
// though there is consensus that this is off (htonll is definitely wrong, and a string is an odd choice), there are
// clients (SyncGateway) that consume the current string, so it can't be changed. Note that only little-endian
// servers are supported for Couchbase, so the 8 byte long inside the string will always be little-endian ordered.
//
// Looks like: "0x000058a71dd25c15"
// Want: 0x155CD21DA7580000 (1539336197457313792 in base10, an epoch time in millionths of a second)
func parseCASToMilliseconds(in string) (int64, error) {
if len(in) < 18 {
logWarnf("Invalid mutation cas value seen in cleanup: %s", in)
return 0, errors.New("invalid cas value provided")
}
offsetIndex := 2 // for the initial "0x"
result := int64(0)
for octetIndex := 7; octetIndex >= 0; octetIndex-- {
char1 := in[offsetIndex+(octetIndex*2)]
char2 := in[offsetIndex+(octetIndex*2)+1]
octet1 := int64(0)
octet2 := int64(0)
if char1 >= 'a' && char1 <= 'f' {
octet1 = int64(char1 - 'a' + 10)
} else if char1 >= 'A' && char1 <= 'F' {
octet1 = int64(char1 - 'A' + 10)
} else if char1 >= '0' && char1 <= '9' {
octet1 = int64(char1 - '0')
} else {
return 0, fmt.Errorf("could not parse CAS: %s", in)
}
if char2 >= 'a' && char2 <= 'f' {
octet2 = int64(char2 - 'a' + 10)
} else if char2 >= 'A' && char2 <= 'F' {
octet2 = int64(char2 - 'A' + 10)
} else if char2 >= '0' && char2 <= '9' {
octet2 = int64(char2 - '0')
} else {
return 0, fmt.Errorf("could not parse CAS: %s", in)
}
result |= octet1 << ((octetIndex * 8) + 4)
result |= octet2 << (octetIndex * 8)
}
// It's in nanoseconds, let's return milliseconds.
return result / 1000000, nil
}
func parseHLCToSeconds(hlc jsonHLC) (int64, error) {
return strconv.ParseInt(hlc.NowSecs, 10, 64)
}
gocbcore-10.2.3/http.go 0000664 0000000 0000000 00000005206 14417540156 0014700 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"context"
"errors"
"io"
"sort"
"sync/atomic"
"time"
)
type httpRequest struct {
Service ServiceType
Endpoint string
Method string
Path string
Username string
Password string
Headers map[string]string
ContentType string
Body []byte
IsIdempotent bool
UniqueID string
Deadline time.Time
RetryStrategy RetryStrategy
RootTraceContext RequestSpanContext
// Whilst the http component will handle deadlines itself this context can be use from places like Ping which
// need to also be able to cancel the context for other reasons.
Context context.Context
CancelFunc context.CancelFunc
User string
retryCount uint32
retryReasons []RetryReason
}
func (hr *httpRequest) retryStrategy() RetryStrategy {
return hr.RetryStrategy
}
func (hr *httpRequest) Cancel() {
if hr.CancelFunc != nil {
hr.CancelFunc()
}
}
func (hr *httpRequest) RetryAttempts() uint32 {
return atomic.LoadUint32(&hr.retryCount)
}
func (hr *httpRequest) Identifier() string {
return hr.UniqueID
}
func (hr *httpRequest) Idempotent() bool {
return hr.IsIdempotent
}
func (hr *httpRequest) RetryReasons() []RetryReason {
return hr.retryReasons
}
func (hr *httpRequest) recordRetryAttempt(reason RetryReason) {
atomic.AddUint32(&hr.retryCount, 1)
idx := sort.Search(len(hr.retryReasons), func(i int) bool {
return hr.retryReasons[i] == reason
})
// if idx is out of the range of retryReasons then it wasn't found.
if idx > len(hr.retryReasons)-1 {
hr.retryReasons = append(hr.retryReasons, reason)
}
}
// HTTPRequest contains the description of an HTTP request to perform.
type HTTPRequest struct {
Service ServiceType
Method string
Endpoint string
Path string
Username string
Password string
Body []byte
Headers map[string]string
ContentType string
IsIdempotent bool
UniqueID string
Deadline time.Time
RetryStrategy RetryStrategy
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// HTTPResponse encapsulates the response from an HTTP request.
type HTTPResponse struct {
Endpoint string
StatusCode int
ContentLength int64
Body io.ReadCloser
}
func wrapHTTPError(req *httpRequest, err error) HTTPError {
if err == nil {
err = errors.New("http error")
}
ierr := HTTPError{
InnerError: err,
}
if req != nil {
ierr.Endpoint = req.Endpoint
ierr.UniqueID = req.UniqueID
ierr.RetryAttempts = req.RetryAttempts()
ierr.RetryReasons = req.RetryReasons()
}
return ierr
}
gocbcore-10.2.3/httpcfgcontroller.go 0000664 0000000 0000000 00000001600 14417540156 0017456 0 ustar 00root root 0000000 0000000 package gocbcore
type httpConfigController struct {
muxer *httpMux
seenNodes map[string]uint64
*baseHTTPConfigController
}
func newHTTPConfigController(bucketName string, props httpPollerProperties, muxer *httpMux,
cfgMgr *configManagementComponent) *httpConfigController {
ctrlr := &httpConfigController{
muxer: muxer,
seenNodes: make(map[string]uint64),
}
ctrlr.baseHTTPConfigController = newBaseHTTPConfigController(bucketName, props, cfgMgr, ctrlr.GetEndpoint)
return ctrlr
}
func (hcc *httpConfigController) GetEndpoint(iterNum uint64) string {
var pickedSrv string
for _, srv := range hcc.muxer.MgmtEps() {
if hcc.seenNodes[srv] >= iterNum {
continue
}
pickedSrv = srv
break
}
if pickedSrv != "" {
hcc.seenNodes[pickedSrv] = iterNum
}
return pickedSrv
}
func (hcc *httpConfigController) CanPoll() bool {
return len(hcc.muxer.MgmtEps()) > 0
}
gocbcore-10.2.3/httpclient_test.go 0000664 0000000 0000000 00000000623 14417540156 0017134 0 ustar 00root root 0000000 0000000 package gocbcore
import "net/http"
func newHTTPComponentWithClient(props httpComponentProps, client *http.Client, muxer *httpMux,
tracer *tracerComponent) *httpComponent {
hc := &httpComponent{
muxer: muxer,
userAgent: props.UserAgent,
defaultRetryStrategy: props.DefaultRetryStrategy,
tracer: tracer,
cli: client,
}
return hc
}
gocbcore-10.2.3/httpclientmux.go 0000664 0000000 0000000 00000002645 14417540156 0016635 0 ustar 00root root 0000000 0000000 package gocbcore
type httpClientMuxEndpoints struct {
capiEpList []routeEndpoint
mgmtEpList []routeEndpoint
n1qlEpList []routeEndpoint
ftsEpList []routeEndpoint
cbasEpList []routeEndpoint
eventingEpList []routeEndpoint
gsiEpList []routeEndpoint
backupEpList []routeEndpoint
}
type httpClientMux struct {
capiEpList []routeEndpoint
mgmtEpList []routeEndpoint
n1qlEpList []routeEndpoint
ftsEpList []routeEndpoint
cbasEpList []routeEndpoint
eventingEpList []routeEndpoint
gsiEpList []routeEndpoint
backupEpList []routeEndpoint
bucket string
uuid string
revID int64
breakerCfg CircuitBreakerConfig
srcConfig routeConfig
tlsConfig *dynTLSConfig
auth AuthProvider
}
func newHTTPClientMux(cfg *routeConfig, endpoints httpClientMuxEndpoints, tlsConfig *dynTLSConfig, auth AuthProvider,
breakerCfg CircuitBreakerConfig) *httpClientMux {
return &httpClientMux{
capiEpList: endpoints.capiEpList,
mgmtEpList: endpoints.mgmtEpList,
n1qlEpList: endpoints.n1qlEpList,
ftsEpList: endpoints.ftsEpList,
cbasEpList: endpoints.cbasEpList,
eventingEpList: endpoints.eventingEpList,
gsiEpList: endpoints.gsiEpList,
backupEpList: endpoints.backupEpList,
bucket: cfg.name,
uuid: cfg.uuid,
revID: cfg.revID,
breakerCfg: breakerCfg,
srcConfig: *cfg,
tlsConfig: tlsConfig,
auth: auth,
}
}
gocbcore-10.2.3/httpcomponent.go 0000664 0000000 0000000 00000041452 14417540156 0016626 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"io"
"io/ioutil"
"math/rand"
"net"
"net/http"
"os"
"sync/atomic"
"syscall"
"time"
"github.com/google/uuid"
)
type httpComponentInterface interface {
DoInternalHTTPRequest(req *httpRequest, skipConfigCheck bool) (*HTTPResponse, error)
}
type httpComponent struct {
cli *http.Client
muxer *httpMux
userAgent string
tracer *tracerComponent
defaultRetryStrategy RetryStrategy
}
type httpComponentProps struct {
UserAgent string
DefaultRetryStrategy RetryStrategy
}
type httpClientProps struct {
connectTimeout time.Duration
maxIdleConns int
maxIdleConnsPerHost int
idleTimeout time.Duration
}
func newHTTPComponent(props httpComponentProps, clientProps httpClientProps, muxer *httpMux, tracer *tracerComponent) *httpComponent {
hc := &httpComponent{
muxer: muxer,
userAgent: props.UserAgent,
defaultRetryStrategy: props.DefaultRetryStrategy,
tracer: tracer,
}
hc.cli = hc.createHTTPClient(clientProps.maxIdleConns, clientProps.maxIdleConnsPerHost, clientProps.idleTimeout,
clientProps.connectTimeout)
return hc
}
func (hc *httpComponent) Close() {
if tsport, ok := hc.cli.Transport.(*http.Transport); ok {
tsport.CloseIdleConnections()
} else {
logDebugf("Could not close idle connections for transport")
}
}
func (hc *httpComponent) DoHTTPRequest(req *HTTPRequest, cb DoHTTPRequestCallback) (PendingOp, error) {
tracer := hc.tracer.StartTelemeteryHandler(metricValueServiceHTTPValue, "http", req.TraceContext)
retryStrategy := hc.defaultRetryStrategy
if req.RetryStrategy != nil {
retryStrategy = req.RetryStrategy
}
ctx, cancel := context.WithCancel(context.Background())
ireq := &httpRequest{
Service: req.Service,
Endpoint: req.Endpoint,
Method: req.Method,
Path: req.Path,
Headers: req.Headers,
ContentType: req.ContentType,
Username: req.Username,
Password: req.Password,
Body: req.Body,
IsIdempotent: req.IsIdempotent,
UniqueID: req.UniqueID,
Deadline: req.Deadline,
RetryStrategy: retryStrategy,
RootTraceContext: tracer.RootContext(),
Context: ctx,
CancelFunc: cancel,
User: req.User,
}
go func() {
resp, err := hc.DoInternalHTTPRequest(ireq, false)
if err != nil {
cancel()
if errors.Is(err, ErrRequestCanceled) {
cb(nil, err)
return
}
tracer.Finish()
cb(nil, wrapHTTPError(ireq, err))
return
}
tracer.Finish()
cb(resp, nil)
}()
return ireq, nil
}
func (hc *httpComponent) DoInternalHTTPRequest(req *httpRequest, skipConfigCheck bool) (*HTTPResponse, error) {
if req.Service == MemdService {
return nil, errInvalidService
}
// This creates a context that has a parent with no cancel function. As such WithCancel will not setup any
// extra go routines and we only need to call cancel on (non-timeout) failure.
ctx := req.Context
if ctx == nil {
ctx = context.Background()
}
ctx, ctxCancel := context.WithCancel(ctx)
// This is easy to do with a bool and defer than to ensure that we cancel after every error.
doneCh := make(chan struct{}, 1)
querySuccess := false
defer func() {
doneCh <- struct{}{}
if !querySuccess {
ctxCancel()
}
}()
start := time.Now()
var cancellationIsTimeout uint32
// Having no deadline is a legitimate case.
if !req.Deadline.IsZero() {
go func() {
select {
case <-time.After(req.Deadline.Sub(start)):
atomic.StoreUint32(&cancellationIsTimeout, 1)
ctxCancel()
case <-doneCh:
}
}()
}
if !skipConfigCheck {
if err := hc.waitForConfig(ctx, req.IsIdempotent, &cancellationIsTimeout); err != nil {
return nil, err
}
}
generator := newHTTPRequestGenerator(ctx, req, hc.userAgent)
var denylist []string
for {
endpoint := req.Endpoint
if endpoint == "" {
var err error
endpoint, err = hc.randomEndpoint(req.Service, denylist)
if err != nil {
return nil, err
}
} else {
err := hc.checkEndpointExists(req.Service, endpoint)
if err != nil {
return nil, err
}
}
var creds []UserPassPair
if req.Username == "" && req.Password == "" {
auth := hc.muxer.Auth()
if auth == nil {
// Shouldn't happen but if it does then probably better to not panic with a nil pointer.
return nil, errCliInternalError
}
var err error
creds, err = auth.Credentials(AuthCredsRequest{
Service: req.Service,
Endpoint: endpoint,
})
if err != nil {
if err := hc.maybeWait(req, CredentialsFetchFailedRetryReason, err, start, endpoint); err != nil {
return nil, err
}
denylist = append(denylist, endpoint)
continue
}
}
hreq, err := generator.NewRequest(endpoint, creds)
if err != nil {
return nil, err
}
dSpan := hc.tracer.StartHTTPDispatchSpan(req, spanNameDispatchToServer)
logSchedf("Writing HTTP request to %s ID=%s", hreq.URL, req.UniqueID)
// we can't close the body of this response as it's long-lived beyond the function
hresp, err := hc.cli.Do(hreq) // nolint: bodyclose
hc.tracer.StopHTTPDispatchSpan(dSpan, hreq, req.UniqueID, req.RetryAttempts())
if err != nil {
logDebugf("Received HTTP Response for ID=%s, errored: %v", req.UniqueID, err)
// Because we don't use the http request context itself to perform timeouts we need to do some translation
// of the error message here for better UX.
if errors.Is(err, context.Canceled) {
isTimeout := atomic.LoadUint32(&cancellationIsTimeout)
if isTimeout == 1 {
if req.IsIdempotent {
err = &TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "http",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: req.retryReasons,
RetryAttempts: req.retryCount,
LastDispatchedTo: endpoint,
}
} else {
err = &TimeoutError{
InnerError: errAmbiguousTimeout,
OperationID: "http",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: req.retryReasons,
RetryAttempts: req.retryCount,
LastDispatchedTo: endpoint,
}
}
} else {
err = errRequestCanceled
}
}
isUserError := false
isUserError = isUserError || errors.Is(err, context.DeadlineExceeded)
isUserError = isUserError || errors.Is(err, context.Canceled)
isUserError = isUserError || errors.Is(err, ErrRequestCanceled)
isUserError = isUserError || errors.Is(err, ErrTimeout)
if isUserError {
return nil, err
}
var retryReason RetryReason
if os.IsTimeout(err) || errors.Is(err, syscall.ECONNREFUSED) {
// Whilst the above comment holds true for once requests are actually sent the dial itself can actually
// timeout, at which point we don't get context canceled.
retryReason = SocketNotAvailableRetryReason
} else if errors.Is(err, io.ErrUnexpectedEOF) {
retryReason = SocketCloseInFlightRetryReason
}
if retryReason == nil {
return nil, err
}
err := hc.maybeWait(req, retryReason, err, start, endpoint)
if err != nil {
return nil, err
}
continue
}
logSchedf("Received HTTP Response for ID=%s, status=%d", req.UniqueID, hresp.StatusCode)
respOut := HTTPResponse{
Endpoint: endpoint,
StatusCode: hresp.StatusCode,
ContentLength: hresp.ContentLength,
Body: hresp.Body,
}
querySuccess = true
return &respOut, nil
}
}
func (hc *httpComponent) waitForConfig(ctx context.Context, isIdempotent bool, cancellationIsTimeout *uint32) error {
for {
revID, err := hc.muxer.ConfigRev()
if err != nil {
return err
}
if revID > -1 {
return nil
}
// We've not successfully been setup with a cluster map yet
select {
case <-ctx.Done():
err := ctx.Err()
if errors.Is(err, context.Canceled) {
isTimeout := atomic.LoadUint32(cancellationIsTimeout)
if isTimeout == 1 {
if isIdempotent {
return errUnambiguousTimeout
}
return errAmbiguousTimeout
}
return errRequestCanceled
}
return err
case <-time.After(500 * time.Microsecond):
}
}
}
func (hc *httpComponent) randomEndpoint(service ServiceType, denylist []string) (string, error) {
var endpoint string
var err error
switch service {
case MgmtService:
endpoint, err = hc.getMgmtEp(denylist)
case CapiService:
endpoint, err = hc.getCapiEp(denylist)
case N1qlService:
endpoint, err = hc.getN1qlEp(denylist)
case FtsService:
endpoint, err = hc.getFtsEp(denylist)
case CbasService:
endpoint, err = hc.getCbasEp(denylist)
case EventingService:
endpoint, err = hc.getEventingEp(denylist)
case GSIService:
endpoint, err = hc.getGSIEp(denylist)
case BackupService:
endpoint, err = hc.getBackupEp(denylist)
}
if err != nil {
return "", err
}
return endpoint, nil
}
func (hc *httpComponent) checkEndpointExists(service ServiceType, endpoint string) error {
var err error
switch service {
case MgmtService:
err = hc.validateEndpoint(endpoint, hc.muxer.MgmtEps())
case CapiService:
err = hc.validateEndpoint(endpoint, hc.muxer.CapiEps())
case N1qlService:
err = hc.validateEndpoint(endpoint, hc.muxer.N1qlEps())
case FtsService:
err = hc.validateEndpoint(endpoint, hc.muxer.FtsEps())
case CbasService:
err = hc.validateEndpoint(endpoint, hc.muxer.CbasEps())
case EventingService:
err = hc.validateEndpoint(endpoint, hc.muxer.EventingEps())
case GSIService:
err = hc.validateEndpoint(endpoint, hc.muxer.GSIEps())
case BackupService:
err = hc.validateEndpoint(endpoint, hc.muxer.BackupEps())
}
if err != nil {
return err
}
return nil
}
func (hc *httpComponent) maybeWait(req *httpRequest, retryReason RetryReason, err error, start time.Time,
endpoint string) error {
shouldRetry, retryTime := retryOrchMaybeRetry(req, retryReason)
if !shouldRetry {
return err
}
select {
case <-time.After(time.Until(retryTime)):
// continue!
case <-time.After(time.Until(req.Deadline)):
if errors.Is(err, context.DeadlineExceeded) {
err = &TimeoutError{
InnerError: errAmbiguousTimeout,
OperationID: "http",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: req.retryReasons,
RetryAttempts: req.retryCount,
LastDispatchedTo: endpoint,
}
}
return err
}
return nil
}
func (hc *httpComponent) getMgmtEp(denylist []string) (string, error) {
endpoints, err := randFromServiceEndpoints(hc.muxer.MgmtEps(), denylist)
return endpoints, err
}
func (hc *httpComponent) getCapiEp(denylist []string) (string, error) {
return randFromServiceEndpoints(hc.muxer.CapiEps(), denylist)
}
func (hc *httpComponent) getN1qlEp(denylist []string) (string, error) {
return randFromServiceEndpoints(hc.muxer.N1qlEps(), denylist)
}
func (hc *httpComponent) getFtsEp(denylist []string) (string, error) {
return randFromServiceEndpoints(hc.muxer.FtsEps(), denylist)
}
func (hc *httpComponent) getCbasEp(denylist []string) (string, error) {
return randFromServiceEndpoints(hc.muxer.CbasEps(), denylist)
}
func (hc *httpComponent) getEventingEp(denylist []string) (string, error) {
return randFromServiceEndpoints(hc.muxer.EventingEps(), denylist)
}
func (hc *httpComponent) getGSIEp(denylist []string) (string, error) {
return randFromServiceEndpoints(hc.muxer.GSIEps(), denylist)
}
func (hc *httpComponent) getBackupEp(denylist []string) (string, error) {
return randFromServiceEndpoints(hc.muxer.BackupEps(), denylist)
}
func (hc *httpComponent) validateEndpoint(endpoint string, endpoints []string) error {
for _, ep := range endpoints {
if ep == endpoint {
return nil
}
}
return errInvalidServer
}
func createTLSConfig(auth AuthProvider, caProvider func() *x509.CertPool) *dynTLSConfig {
return &dynTLSConfig{
BaseConfig: &tls.Config{
GetClientCertificate: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert, err := auth.Certificate(AuthCertRequest{})
if err != nil {
return nil, err
}
if cert == nil {
return &tls.Certificate{}, nil
}
return cert, nil
},
MinVersion: tls.VersionTLS12,
},
Provider: caProvider,
}
}
func (hc *httpComponent) createHTTPClient(maxIdleConns, maxIdleConnsPerHost int, idleTimeout time.Duration, connectTimeout time.Duration) *http.Client {
httpDialer := &net.Dialer{
Timeout: connectTimeout,
KeepAlive: 30 * time.Second,
}
// We set ForceAttemptHTTP2, which will update the base-config to support HTTP2
// automatically, so that all configs from it will look for that.
httpTransport := &http.Transport{
ForceAttemptHTTP2: true,
Dial: func(network, addr string) (net.Conn, error) {
return httpDialer.Dial(network, addr)
},
DialTLS: func(network, addr string) (net.Conn, error) {
tcpConn, err := httpDialer.Dial(network, addr)
if err != nil {
return nil, err
}
// We set up the transport to point at the BaseConfig from the dynamic TLS system.
httpTLSConfig := hc.muxer.Get().tlsConfig
if httpTLSConfig == nil {
return nil, errors.New("TLS is not configured on this Agent")
}
srvTLSConfig, err := httpTLSConfig.MakeForAddr(addr)
if err != nil {
return nil, err
}
tlsConn := tls.Client(tcpConn, srvTLSConfig)
return tlsConn, nil
},
MaxIdleConns: maxIdleConns,
MaxIdleConnsPerHost: maxIdleConnsPerHost,
IdleConnTimeout: idleTimeout,
}
httpCli := &http.Client{
Transport: httpTransport,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
// All that we're doing here is setting auth on any redirects.
// For that reason we can just pull it off the oldest (first) request.
if len(via) >= 10 {
// Just duplicate the default behaviour for maximum redirects.
return errors.New("stopped after 10 redirects")
}
oldest := via[0]
auth := oldest.Header.Get("Authorization")
if auth != "" {
req.Header.Set("Authorization", auth)
}
return nil
},
}
return httpCli
}
/* #nosec G404 */
func randFromServiceEndpoints(endpoints []string, denylist []string) (string, error) {
var allowList []string
for _, ep := range endpoints {
if inDenyList(ep, denylist) {
continue
}
allowList = append(allowList, ep)
}
if len(allowList) == 0 {
return "", errServiceNotAvailable
}
return allowList[rand.Intn(len(allowList))], nil
}
func inDenyList(ep string, denylist []string) bool {
for _, b := range denylist {
if ep == b {
return true
}
}
return false
}
func injectJSONCreds(body []byte, creds []UserPassPair) []byte {
var props map[string]json.RawMessage
err := json.Unmarshal(body, &props)
if err == nil {
if _, ok := props["creds"]; ok {
// Early out if the user has already passed a set of credentials.
return body
}
jsonCreds, err := json.Marshal(creds)
if err == nil {
props["creds"] = json.RawMessage(jsonCreds)
newBody, err := json.Marshal(props)
if err == nil {
return newBody
}
}
}
return body
}
type httpRequestGenerator struct {
ctx context.Context
request *httpRequest
header http.Header
}
func newHTTPRequestGenerator(ctx context.Context, req *httpRequest, userAgent string) *httpRequestGenerator {
header := make(http.Header)
if req.ContentType != "" {
header.Set("Content-Type", req.ContentType)
} else {
header.Set("Content-Type", "application/json")
}
if len(req.User) > 0 {
header.Set("cb-on-behalf-of", req.User)
}
for key, val := range req.Headers {
header.Set(key, val)
}
var uniqueID string
if req.UniqueID != "" {
uniqueID = req.UniqueID
} else {
uniqueID = uuid.New().String()
}
header.Set("User-Agent", clientInfoString(uniqueID, userAgent))
return &httpRequestGenerator{
ctx: ctx,
request: req,
header: header,
}
}
func (hrg *httpRequestGenerator) NewRequest(endpoint string, creds []UserPassPair) (*http.Request, error) {
// Generate a request URI
reqURI := endpoint + hrg.request.Path
hreq, err := http.NewRequestWithContext(hrg.ctx, hrg.request.Method, reqURI, nil)
if err != nil {
return nil, err
}
hreq.Header = hrg.header
body := hrg.request.Body
// Inject credentials into the request
if hrg.request.Username != "" || hrg.request.Password != "" {
hreq.SetBasicAuth(hrg.request.Username, hrg.request.Password)
} else {
if hrg.request.Service == N1qlService || hrg.request.Service == CbasService ||
hrg.request.Service == FtsService {
// Handle service which support multi-bucket authentication using
// injection into the body of the request.
if len(creds) == 1 {
hreq.SetBasicAuth(creds[0].Username, creds[0].Password)
} else {
body = injectJSONCreds(body, creds)
}
} else {
if len(creds) != 1 {
return nil, errInvalidCredentials
}
hreq.SetBasicAuth(creds[0].Username, creds[0].Password)
}
}
hreq.Body = ioutil.NopCloser(bytes.NewReader(body))
return hreq, nil
}
gocbcore-10.2.3/httpmux.go 0000664 0000000 0000000 00000015110 14417540156 0015425 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"bytes"
"fmt"
"net/url"
"sync/atomic"
"unsafe"
)
type httpMux struct {
muxPtr unsafe.Pointer
breakerCfg CircuitBreakerConfig
cfgMgr configManager
noSeedNodeTLS bool
}
func newHTTPMux(breakerCfg CircuitBreakerConfig, cfgMgr configManager, muxState *httpClientMux, noSeedNodeTLS bool) *httpMux {
mux := &httpMux{
breakerCfg: breakerCfg,
cfgMgr: cfgMgr,
muxPtr: unsafe.Pointer(muxState),
noSeedNodeTLS: noSeedNodeTLS,
}
cfgMgr.AddConfigWatcher(mux)
return mux
}
func (mux *httpMux) Get() *httpClientMux {
return (*httpClientMux)(atomic.LoadPointer(&mux.muxPtr))
}
func (mux *httpMux) Update(old, new *httpClientMux) bool {
if new == nil {
logErrorf("Attempted to update to nil httpClientMux")
return false
}
if old != nil {
return atomic.CompareAndSwapPointer(&mux.muxPtr, unsafe.Pointer(old), unsafe.Pointer(new))
}
if atomic.SwapPointer(&mux.muxPtr, unsafe.Pointer(new)) != nil {
logErrorf("Updated from nil attempted on initialized httpClientMux")
return false
}
return true
}
func (mux *httpMux) Clear() *httpClientMux {
val := atomic.SwapPointer(&mux.muxPtr, nil)
return (*httpClientMux)(val)
}
func (mux *httpMux) OnNewRouteConfig(cfg *routeConfig) {
oldHTTPMux := mux.Get()
endpoints := mux.buildEndpoints(cfg, oldHTTPMux.tlsConfig != nil)
var buffer bytes.Buffer
addEps := func(title string, eps []routeEndpoint) {
fmt.Fprintf(&buffer, "%s Eps:\n", title)
for _, ep := range eps {
fmt.Fprintf(&buffer, " - %s\n", ep.Address)
}
}
buffer.WriteString(fmt.Sprintln("HTTP muxer applying endpoints:"))
buffer.WriteString(fmt.Sprintf("Bucket: %s\n", cfg.name))
addEps("Capi", endpoints.capiEpList)
addEps("Mgmt", endpoints.mgmtEpList)
addEps("N1ql", endpoints.n1qlEpList)
addEps("FTS", endpoints.ftsEpList)
addEps("CBAS", endpoints.cbasEpList)
addEps("Eventing", endpoints.eventingEpList)
addEps("GSI", endpoints.gsiEpList)
addEps("Backup", endpoints.backupEpList)
logDebugf(buffer.String())
newHTTPMux := newHTTPClientMux(cfg, endpoints, oldHTTPMux.tlsConfig, oldHTTPMux.auth, mux.breakerCfg)
if !mux.Update(oldHTTPMux, newHTTPMux) {
logDebugf("Failed to update HTTP mux")
}
}
func (mux *httpMux) UpdateTLS(tlsConfig *dynTLSConfig, auth AuthProvider) {
oldMux := mux.Get()
endpoints := mux.buildEndpoints(&oldMux.srcConfig, tlsConfig != nil)
newMux := newHTTPClientMux(&oldMux.srcConfig, endpoints, tlsConfig, auth, oldMux.breakerCfg)
if !atomic.CompareAndSwapPointer(&mux.muxPtr, unsafe.Pointer(oldMux), unsafe.Pointer(newMux)) {
// A new config must have come in so let's try again.
mux.UpdateTLS(tlsConfig, auth)
}
}
func makeEpList(endpoints []routeEndpoint) []string {
var epList []string
for _, ep := range endpoints {
epList = append(epList, ep.Address)
}
return epList
}
// CapiEps returns the capi endpoints with the path escaped bucket name appended.
func (mux *httpMux) CapiEps() []string {
clientMux := mux.Get()
if clientMux == nil {
return nil
}
var epList []string
for _, ep := range clientMux.capiEpList {
epList = append(epList, ep.Address+"/"+url.PathEscape(clientMux.bucket))
}
return epList
}
func (mux *httpMux) MgmtEps() []string {
clientMux := mux.Get()
if clientMux == nil {
return nil
}
return makeEpList(clientMux.mgmtEpList)
}
func (mux *httpMux) N1qlEps() []string {
clientMux := mux.Get()
if clientMux == nil {
return nil
}
return makeEpList(clientMux.n1qlEpList)
}
func (mux *httpMux) CbasEps() []string {
clientMux := mux.Get()
if clientMux == nil {
return nil
}
return makeEpList(clientMux.cbasEpList)
}
func (mux *httpMux) FtsEps() []string {
clientMux := mux.Get()
if clientMux == nil {
return nil
}
return makeEpList(clientMux.ftsEpList)
}
func (mux *httpMux) EventingEps() []string {
if cMux := mux.Get(); cMux != nil {
return makeEpList(cMux.eventingEpList)
}
return nil
}
func (mux *httpMux) GSIEps() []string {
if cMux := mux.Get(); cMux != nil {
return makeEpList(cMux.gsiEpList)
}
return nil
}
func (mux *httpMux) BackupEps() []string {
if cMux := mux.Get(); cMux != nil {
return makeEpList(cMux.backupEpList)
}
return nil
}
func (mux *httpMux) ConfigRev() (int64, error) {
clientMux := mux.Get()
if clientMux == nil {
return 0, errShutdown
}
return clientMux.revID, nil
}
func (mux *httpMux) Close() error {
mux.cfgMgr.RemoveConfigWatcher(mux)
mux.Clear()
return nil
}
func (mux *httpMux) Auth() AuthProvider {
clientMux := mux.Get()
if clientMux == nil {
return nil
}
return clientMux.auth
}
func (mux *httpMux) buildEndpoints(config *routeConfig, useTLS bool) httpClientMuxEndpoints {
var endpoints httpClientMuxEndpoints
if useTLS {
if mux.noSeedNodeTLS {
endpoints = httpClientMuxEndpoints{
capiEpList: mux.buildSSLEpListWithNoSSLSeed(config.capiEpList),
mgmtEpList: mux.buildSSLEpListWithNoSSLSeed(config.mgmtEpList),
n1qlEpList: mux.buildSSLEpListWithNoSSLSeed(config.n1qlEpList),
ftsEpList: mux.buildSSLEpListWithNoSSLSeed(config.ftsEpList),
cbasEpList: mux.buildSSLEpListWithNoSSLSeed(config.cbasEpList),
eventingEpList: mux.buildSSLEpListWithNoSSLSeed(config.eventingEpList),
gsiEpList: mux.buildSSLEpListWithNoSSLSeed(config.gsiEpList),
backupEpList: mux.buildSSLEpListWithNoSSLSeed(config.backupEpList),
}
} else {
endpoints = httpClientMuxEndpoints{
capiEpList: config.capiEpList.SSLEndpoints,
mgmtEpList: config.mgmtEpList.SSLEndpoints,
n1qlEpList: config.n1qlEpList.SSLEndpoints,
ftsEpList: config.ftsEpList.SSLEndpoints,
cbasEpList: config.cbasEpList.SSLEndpoints,
eventingEpList: config.eventingEpList.SSLEndpoints,
gsiEpList: config.gsiEpList.SSLEndpoints,
backupEpList: config.backupEpList.SSLEndpoints,
}
}
} else {
endpoints = httpClientMuxEndpoints{
capiEpList: config.capiEpList.NonSSLEndpoints,
mgmtEpList: config.mgmtEpList.NonSSLEndpoints,
n1qlEpList: config.n1qlEpList.NonSSLEndpoints,
ftsEpList: config.ftsEpList.NonSSLEndpoints,
cbasEpList: config.cbasEpList.NonSSLEndpoints,
eventingEpList: config.eventingEpList.NonSSLEndpoints,
gsiEpList: config.gsiEpList.NonSSLEndpoints,
backupEpList: config.backupEpList.NonSSLEndpoints,
}
}
return endpoints
}
func (mux *httpMux) buildSSLEpListWithNoSSLSeed(list routeEndpoints) []routeEndpoint {
var newlist []routeEndpoint
for _, ep := range list.SSLEndpoints {
if !ep.IsSeedNode {
newlist = append(newlist, ep)
}
}
for _, ep := range list.NonSSLEndpoints {
if ep.IsSeedNode {
newlist = append(newlist, ep)
break
}
}
return newlist
}
gocbcore-10.2.3/jcbmock/ 0000775 0000000 0000000 00000000000 14417540156 0014777 5 ustar 00root root 0000000 0000000 gocbcore-10.2.3/jcbmock/commands.go 0000664 0000000 0000000 00000002353 14417540156 0017132 0 ustar 00root root 0000000 0000000 package jcbmock
import (
"encoding/json"
"log"
"strings"
)
type command struct {
Code CmdCode
Body map[string]interface{}
}
func (c command) Encode() (encoded []byte) {
payload := make(map[string]interface{})
payload["command"] = c.Code
if c.Body != nil {
payload["payload"] = c.Body
}
encoded, err := json.Marshal(payload)
if err != nil {
panic("Received invalid command for marshal")
}
return
}
func (c command) Set(key string, value interface{}) {
c.Body[key] = value
}
// Command is used to specify a command to run.
type Command interface {
Encode() []byte
Set(key string, value interface{})
}
// Response is the result of running a command.
type Response struct {
Payload map[string]interface{}
}
// Success returns whether or not the command was successful.
func (r *Response) Success() bool {
s, exists := r.Payload["status"]
if !exists {
log.Print("Warning: status field not found!")
return false
}
b, castok := s.(string)
if !castok {
log.Print("Bad type in 'status'")
return false
}
return strings.ToLower(b)[0] == 'o'
}
// NewCommand returns a new command for a given command code and body.
func NewCommand(code CmdCode, body map[string]interface{}) Command {
return command{Code: code, Body: body}
}
gocbcore-10.2.3/jcbmock/constants.go 0000664 0000000 0000000 00000005723 14417540156 0017351 0 ustar 00root root 0000000 0000000 package jcbmock
// BucketType represents the type of bucket to use.
type BucketType int
const (
// BCouchbase represents to use a couchbase bucket.
BCouchbase BucketType = 0
// BMemcached represents to use a memcached bucket.
BMemcached = iota
)
// CmdCode represents a command code to send to Couchbase Mock.
type CmdCode string
const (
// CFailover represents a command code to send FAILOVER to the mock.
CFailover CmdCode = "FAILOVER"
// CRespawn represents a command code to send RESPAWN to the mock.
CRespawn = "RESPAWN"
// CHiccup represents a command code to send HICCUP to the mock.
CHiccup = "HICCUP"
// CTruncate represents a command code to send TRUNCATE to the mock.
CTruncate = "TRUNCATE"
// CMockinfo represents a command code to send MOCKINFO to the mock.
CMockinfo = "MOCKINFO"
// CPersist represents a command code to send PERSIST to the mock.
CPersist = "PERSIST"
// CCache represents a command code to send CACHE to the mock.
CCache = "CACHE"
// CUnpersist represents a command code to send UNPERSIST to the mock.
CUnpersist = "UNPERSIST"
// CUncache represents a command code to send UNCACHE to the mock.
CUncache = "UNCACHE"
// CEndure represents a command code to send ENDURE to the mock.
CEndure = "ENDURE"
// CPurge represents a command code to send PURGE to the mock.
CPurge = "PURGE"
// CKeyinfo represents a command code to send KEYINFO to the mock.
CKeyinfo = "KEYINFO"
// CTimeTravel represents a command code to send TIME_TRAVEL to the mock.
CTimeTravel = "TIME_TRAVEL"
// CHelp represents a command code to send HELP to the mock.
CHelp = "HELP"
// COpFail represents a command code to send OPFAIL to the mock.
COpFail = "OPFAIL"
// CSetCCCP represents a command code to SET_CCCP the mock.
CSetCCCP = "SET_CCCP"
// CGetMcPorts represents a command code to GET_MCPORTS the mock.
CGetMcPorts = "GET_MCPORTS"
// CRegenVBCoords represents a command code to REGEN_VBCOORDS the mock.
CRegenVBCoords = "REGEN_VBCOORDS"
// CResetQueryState represents a command code to RESET_QUERYSTATE the mock.
CResetQueryState = "RESET_QUERYSTATE"
// CStartCmdLog represents a command code to START_CMDLOG the mock.
CStartCmdLog = "START_CMDLOG"
// CStopCmdLog represents a command code to STOP_CMDLOG the mock.
CStopCmdLog = "STOP_CMDLOG"
// CGetCmdLog represents a command code to GET_CMDLOG the mock.
CGetCmdLog = "GET_CMDLOG"
// CStartRetryVerify represents a command code to START_RETRY_VERIFY the mock.
CStartRetryVerify = "START_RETRY_VERIFY"
// CCheckRetryVerify represents a command code to CHECK_RETRY_VERIFY the mock.
CCheckRetryVerify = "CHECK_RETRY_VERIFY"
// CSetEnhancedErrors represents a command code to SET_ENHANCED_ERRORS the mock.
CSetEnhancedErrors = "SET_ENHANCED_ERRORS"
// CSetCompression represents a command code to SET_COMPRESSION the mock.
CSetCompression = "SET_COMPRESSION"
// CSetSASLMechanisms represents a command code to SET_SASL_MECHANISMS the mock.
CSetSASLMechanisms = "SET_SASL_MECHANISMS"
)
gocbcore-10.2.3/jcbmock/downloader.go 0000664 0000000 0000000 00000003564 14417540156 0017474 0 ustar 00root root 0000000 0000000 package jcbmock
import (
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
)
// Downloads and caches the mock server, so that it is retrievable
// for automatic testing
const defaultMockVersion = "1.5.25"
const defaultMockFile = "CouchbaseMock-" + defaultMockVersion + ".jar"
const defaultMockUrl = "https://packages.couchbase.com/clients/c/mock/" + defaultMockFile
// GetMockPath ensures that the mock path is available
func GetMockPath() (path string, err error) {
var url string
if path = os.Getenv("GOCB_MOCK_PATH"); path == "" {
path = strings.Join([]string{os.TempDir(), defaultMockFile}, string(os.PathSeparator))
}
if url = os.Getenv("GOCB_MOCK_URL"); url == "" {
url = defaultMockUrl
}
path, err = filepath.Abs(path)
if err != nil {
throwMockError("Couldn't get absolute path (!)", err)
}
info, err := os.Stat(path)
if err == nil && info.Size() > 0 {
return path, nil
} else if err != nil && !os.IsNotExist(err) {
throwMockError("Couldn't resolve existing path", err)
}
if err := os.Remove(path); err != nil {
log.Printf("Couldn't remove existing mock: %v", err)
}
log.Printf("Downloading %s to %s", url, path)
resp, err := http.Get(defaultMockUrl)
if err != nil {
throwMockError("Couldn't create HTTP request (or other error)", err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Printf("Failed to close response body: %v", err)
}
}()
if resp.StatusCode != 200 {
throwMockError(fmt.Sprintf("Got HTTP %d from URL", resp.StatusCode), errors.New("non-200 response"))
}
out, err := os.Create(path)
if err != nil {
throwMockError("Couldn't open output file", err)
}
defer func() {
if err := out.Close(); err != nil {
log.Printf("Failed to close file: %v", err)
}
}()
_, err = io.Copy(out, resp.Body)
if err != nil {
throwMockError("Couldn't write response", err)
}
return path, nil
}
gocbcore-10.2.3/jcbmock/mock.go 0000664 0000000 0000000 00000014337 14417540156 0016267 0 ustar 00root root 0000000 0000000 package jcbmock
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"log"
"net"
"os"
"os/exec"
"strconv"
"strings"
"time"
)
type mockError struct {
cause error
message string
}
func (e mockError) Error() string {
return fmt.Sprintf("Mock Error: %s (caused by %s)", e.message, e.cause.Error())
}
func throwMockError(msg string, cause error) {
if cause == nil {
cause = errors.New("no cause")
}
panic(mockError{message: msg, cause: cause})
}
const mockInitTimeout = 5 * time.Second
// BucketSpec describes the specification of a bucket.
type BucketSpec struct {
// Type of the bucket
Type BucketType
// Name of the bucket
Name string
// Password for the bucket (empty means no password)
Password string
}
func (s BucketSpec) toString() string {
specArr := make([]string, 3)
specArr[0] = s.Name
specArr[1] = s.Password
if s.Type == BCouchbase {
specArr[2] = "couchbase"
} else {
specArr[2] = "memcache"
}
return strings.Join(specArr, ":")
}
// Mock is used for mocking a Couchbase Server cluster.
type Mock struct {
// Executable object (for termination)
cmd *exec.Cmd
// Connection to the mock itself
conn net.Conn
// List of ports for the bucket
EntryPort uint16
// Internal reader-writer
rw *bufio.ReadWriter
}
// Close closes the mock and kills the underlying process
func (m *Mock) Close() {
log.Printf("Closing mock %p\n", m)
if m.cmd != nil && m.cmd.Process != nil {
if err := m.cmd.Process.Kill(); err != nil {
log.Printf("Error killing process: %v", err)
return
}
_, err := m.cmd.Process.Wait()
if err != nil {
log.Printf("Error waiting for process to end: %v", err)
return
}
}
if m.conn != nil {
if err := m.conn.Close(); err != nil {
log.Printf("Error closing connection: %v", err)
}
}
m.EntryPort = 0
}
// Control sends a control command to the mock.
func (m *Mock) Control(c Command) (r Response) {
reqbytes := c.Encode()
reqbytes = append(reqbytes, '\n')
if _, err := m.rw.Write(reqbytes); err != nil {
throwMockError("Short write while sending command", err)
}
if err := m.rw.Flush(); err != nil {
throwMockError("Short write while flushing command to writer", err)
}
log.Printf("Sent '%s'", reqbytes[:len(reqbytes)-1])
resbytes, err := m.rw.ReadBytes('\n')
log.Printf("Got '%s'", resbytes[:len(resbytes)-1])
if err != nil {
throwMockError("Short read while receiving response", err)
}
r = Response{Payload: make(map[string]interface{})}
if err := json.Unmarshal(resbytes, &r.Payload); err != nil {
throwMockError("Couldn't decode response JSON", err)
}
return r
}
// MemcachedPorts returns the list of memcached ports that this mock is listening on.
func (m *Mock) MemcachedPorts() (out []uint16) {
c := NewCommand(CGetMcPorts, nil)
r := m.Control(c)
if !r.Success() {
throwMockError("Couldn't get memcached ports!", nil)
}
arr, ok := r.Payload["payload"].([]interface{})
if !ok {
throwMockError("Badly formatted port array", nil)
}
out = make([]uint16, 0)
for _, v := range arr {
tmpV, ok := v.(float64)
if !ok {
throwMockError(fmt.Sprintf("Expected numeric value. Got %T", v), nil)
}
out = append(out, uint16(tmpV))
}
return
}
// Version returns the version of this mock.
func (m *Mock) Version() string {
return defaultMockVersion
}
func (m *Mock) buildSpecStrings(specs []BucketSpec) string {
var strSpecs []string
for _, spec := range specs {
strSpecs = append(strSpecs, spec.toString())
}
return strings.Join(strSpecs, ",")
}
// NewMock creates and runs a new mock instance
// The path is the path to the mock jar.
// nodes is the total number of cluster nodes (and thus the number of mock threads)
// replicas is the number of replica nodes (subset of the number of nodes) for each couchbase bucket.
// vbuckets is the number of vbuckets to use for each couchbase bucket
// specs should be a list of specifications of buckets to use..
func NewMock(path string, nodes uint, replicas uint, vbuckets uint, specs ...BucketSpec) (m *Mock, err error) {
var lsn *net.TCPListener
chAccept := make(chan bool)
m = &Mock{}
defer func() {
close(chAccept)
if lsn != nil {
if err := lsn.Close(); err != nil {
log.Printf("Failed to close listener: %v", err)
}
}
exc := recover()
if exc == nil {
// No errors, everything is OK
return
}
// Close mock on error, destroying resources
m.Close()
if mExc, ok := exc.(mockError); !ok {
panic(mExc)
} else {
m = nil
err = mExc
}
}()
if lsn, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0}); err != nil {
throwMockError("Couldn't set up listening socket", err)
}
_, ctlPort, err := net.SplitHostPort(lsn.Addr().String())
if err != nil {
log.Fatalf("Failed to split host and port: %v", err)
}
log.Printf("Listening for control connection at %s\n", ctlPort)
go func() {
var err error
defer func() {
chAccept <- false
}()
if m.conn, err = lsn.Accept(); err != nil {
throwMockError("Couldn't accept incoming control connection from mock", err)
return
}
}()
if len(specs) == 0 {
specs = []BucketSpec{{Name: "default", Type: BCouchbase}}
}
options := []string{
"-jar", path, "--harakiri-monitor", "localhost:" + ctlPort, "--port", "0",
"--replicas", strconv.Itoa(int(replicas)),
"--vbuckets", strconv.Itoa(int(vbuckets)),
"--nodes", strconv.Itoa(int(nodes)),
"--buckets", m.buildSpecStrings(specs),
}
log.Printf("Invoking java %s", strings.Join(options, " "))
m.cmd = exec.Command("java", options...)
m.cmd.Stdout = os.Stdout
m.cmd.Stderr = os.Stderr
if err = m.cmd.Start(); err != nil {
m.cmd = nil
throwMockError("Couldn't start command", err)
}
select {
case <-chAccept:
break
case <-time.After(mockInitTimeout):
throwMockError("Timed out waiting for initialization", errors.New("timeout"))
}
m.rw = bufio.NewReadWriter(bufio.NewReader(m.conn), bufio.NewWriter(m.conn))
// Read the port buffer, which is delimited by a NUL byte
if portBytes, err := m.rw.ReadBytes(0); err != nil {
throwMockError("Couldn't get port information", err)
} else {
portBytes = portBytes[:len(portBytes)-1]
if entryPort, err := strconv.Atoi(string(portBytes)); err != nil {
throwMockError("Incorrectly formatted port from mock", err)
} else {
m.EntryPort = uint16(entryPort)
}
}
log.Printf("Mock HTTP port at %d\n", m.EntryPort)
return
}
gocbcore-10.2.3/ketama.go 0000664 0000000 0000000 00000005417 14417540156 0015167 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"crypto/md5" // nolint: gosec
"fmt"
"sort"
)
// "Point" in the ring hash entry. See lcbvb_CONTINUUM
type routeKetamaContinuum struct {
index uint32
point uint32
}
type ketamaSorter struct {
elems []routeKetamaContinuum
}
func (c ketamaSorter) Len() int { return len(c.elems) }
func (c ketamaSorter) Swap(i, j int) { c.elems[i], c.elems[j] = c.elems[j], c.elems[i] }
func (c ketamaSorter) Less(i, j int) bool { return c.elems[i].point < c.elems[j].point }
type ketamaContinuum struct {
entries []routeKetamaContinuum
}
func ketamaHash(key []byte) uint32 {
digest := md5.Sum(key) // nolint: gosec
return ((uint32(digest[3])&0xFF)<<24 |
(uint32(digest[2])&0xFF)<<16 |
(uint32(digest[1])&0xFF)<<8 |
(uint32(digest[0]) & 0xFF)) & 0xffffffff
}
func newKetamaContinuum(endpointList []routeEndpoint) *ketamaContinuum {
continuum := ketamaContinuum{}
var serverList []string
for _, s := range endpointList {
serverList = append(serverList, s.Address)
}
// Libcouchbase presorts this. Might not strictly be required..
sort.Strings(serverList)
for ss, authority := range serverList {
// 160 points per server
for hh := 0; hh < 40; hh++ {
hostkey := []byte(fmt.Sprintf("%s-%d", authority, hh))
digest := md5.Sum(hostkey) // nolint: gosec
for nn := 0; nn < 4; nn++ {
var d1 = uint32(digest[3+nn*4]&0xff) << 24
var d2 = uint32(digest[2+nn*4]&0xff) << 16
var d3 = uint32(digest[1+nn*4]&0xff) << 8
var d4 = uint32(digest[0+nn*4] & 0xff)
var point = d1 | d2 | d3 | d4
continuum.entries = append(continuum.entries, routeKetamaContinuum{
point: point,
index: uint32(ss),
})
}
}
}
sort.Sort(ketamaSorter{continuum.entries})
return &continuum
}
func (continuum ketamaContinuum) IsValid() bool {
return len(continuum.entries) > 0
}
func (continuum ketamaContinuum) nodeByHash(hash uint32) (int, error) {
var lowp = uint32(0)
var highp = uint32(len(continuum.entries))
var maxp = highp
if len(continuum.entries) <= 0 {
logErrorf("0-length ketama map! Mapping to node 0.")
return 0, errCliInternalError
}
// Copied from libcouchbase vbucket.c (map_ketama)
for {
midp := lowp + (highp-lowp)/2
if midp == maxp {
// Roll over to first entry
return int(continuum.entries[0].index), nil
}
mid := continuum.entries[midp].point
var prev uint32
if midp == 0 {
prev = 0
} else {
prev = continuum.entries[midp-1].point
}
if hash <= mid && hash > prev {
return int(continuum.entries[midp].index), nil
}
if mid < hash {
lowp = midp + 1
} else {
highp = midp - 1
}
if lowp > highp {
return int(continuum.entries[0].index), nil
}
}
}
func (continuum ketamaContinuum) NodeByKey(key []byte) (int, error) {
return continuum.nodeByHash(ketamaHash(key))
}
gocbcore-10.2.3/kvmux.go 0000664 0000000 0000000 00000056505 14417540156 0015103 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"bytes"
"container/list"
"errors"
"fmt"
"io"
"sort"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/couchbase/gocbcore/v10/memd"
)
type bucketCapabilityVerifier interface {
HasBucketCapabilityStatus(cap BucketCapability, status BucketCapabilityStatus) bool
}
type dispatcher interface {
DispatchDirect(req *memdQRequest) (PendingOp, error)
RequeueDirect(req *memdQRequest, isRetry bool)
DispatchDirectToAddress(req *memdQRequest, pipeline *memdPipeline) (PendingOp, error)
CollectionsEnabled() bool
SupportsCollections() bool
SetPostCompleteErrorHandler(handler postCompleteErrorHandler)
PipelineSnapshot() (*pipelineSnapshot, error)
}
type kvMux struct {
muxPtr unsafe.Pointer
collectionsEnabled bool
queueSize int
poolSize int
cfgMgr *configManagementComponent
errMapMgr *errMapComponent
tracer *tracerComponent
dialer *memdClientDialerComponent
postCompleteErrHandler postCompleteErrorHandler
// muxStateWriteLock is necessary for functions which update the muxPtr, due to the scenario where ForceReconnect and
// OnNewRouteConfig could race. ForceReconnect must succeed and cannot fail because OnNewRouteConfig has updated
// the mux state whilst force is attempting to update it. We could also end up in a situation where a full reconnect
// is occurring at the same time as a pipeline takeover and scenarios like that, including missing a config update because
// ForceReconnect has won the race.
// There is no need for read side locks as we are locking around an atomic and it is only the write sides that present
// a potential issue.
muxStateWriteLock sync.Mutex
shutdownSig chan struct{}
clientCloseWg sync.WaitGroup
noTLSSeedNode bool
hasSeenConfigCh chan struct{}
}
type kvMuxProps struct {
CollectionsEnabled bool
QueueSize int
PoolSize int
NoTLSSeedNode bool
}
func newKVMux(props kvMuxProps, cfgMgr *configManagementComponent, errMapMgr *errMapComponent, tracer *tracerComponent,
dialer *memdClientDialerComponent, muxState *kvMuxState) *kvMux {
mux := &kvMux{
queueSize: props.QueueSize,
poolSize: props.PoolSize,
collectionsEnabled: props.CollectionsEnabled,
cfgMgr: cfgMgr,
errMapMgr: errMapMgr,
tracer: tracer,
dialer: dialer,
shutdownSig: make(chan struct{}),
noTLSSeedNode: props.NoTLSSeedNode,
muxPtr: unsafe.Pointer(muxState),
hasSeenConfigCh: make(chan struct{}),
}
cfgMgr.AddConfigWatcher(mux)
return mux
}
func (mux *kvMux) getState() *kvMuxState {
muxPtr := atomic.LoadPointer(&mux.muxPtr)
if muxPtr == nil {
return nil
}
return (*kvMuxState)(muxPtr)
}
func (mux *kvMux) updateState(old, new *kvMuxState) bool {
if new == nil {
logErrorf("Attempted to update to nil kvMuxState")
return false
}
if old != nil {
return atomic.CompareAndSwapPointer(&mux.muxPtr, unsafe.Pointer(old), unsafe.Pointer(new))
}
if atomic.SwapPointer(&mux.muxPtr, unsafe.Pointer(new)) != nil {
logErrorf("Updated from nil attempted on initialized kvMuxState")
return false
}
return true
}
func (mux *kvMux) clear() *kvMuxState {
mux.muxStateWriteLock.Lock()
val := atomic.SwapPointer(&mux.muxPtr, nil)
mux.muxStateWriteLock.Unlock()
return (*kvMuxState)(val)
}
func (mux *kvMux) OnNewRouteConfig(cfg *routeConfig) {
mux.muxStateWriteLock.Lock()
defer mux.muxStateWriteLock.Unlock()
oldMuxState := mux.getState()
newMuxState := mux.newKVMuxState(cfg, oldMuxState.tlsConfig, oldMuxState.authMechanisms, oldMuxState.auth)
// Attempt to atomically update the routing data
if !mux.updateState(oldMuxState, newMuxState) {
logWarnf("Someone preempted the config update, skipping update")
return
}
if oldMuxState.RevID() == -1 && newMuxState.RevID() > -1 {
if cfg.name != "" && mux.collectionsEnabled && !newMuxState.collectionsSupported {
logDebugf("Collections disabled as unsupported")
}
close(mux.hasSeenConfigCh)
}
if !mux.collectionsEnabled {
// If collections just aren't enabled then we never need to refresh the connections because collections
// have come online.
mux.pipelineTakeover(oldMuxState, newMuxState)
} else if oldMuxState.RevID() == -1 || oldMuxState.collectionsSupported == newMuxState.collectionsSupported {
// Get the new muxer to takeover the pipelines from the older one
mux.pipelineTakeover(oldMuxState, newMuxState)
} else {
// Collections support has changed so we need to reconnect all connections in order to support the new
// state.
mux.reconnectPipelines(oldMuxState, newMuxState, true)
}
mux.requeueRequests(oldMuxState)
}
func (mux *kvMux) SetPostCompleteErrorHandler(handler postCompleteErrorHandler) {
mux.postCompleteErrHandler = handler
}
func (mux *kvMux) ConfigRev() (int64, error) {
clientMux := mux.getState()
if clientMux == nil {
return 0, errShutdown
}
return clientMux.RevID(), nil
}
func (mux *kvMux) ConfigUUID() string {
clientMux := mux.getState()
if clientMux == nil {
return ""
}
return clientMux.UUID()
}
func (mux *kvMux) KeyToVbucket(key []byte) (uint16, error) {
clientMux := mux.getState()
if clientMux == nil || clientMux.VBMap() == nil {
return 0, errShutdown
}
return clientMux.VBMap().VbucketByKey(key), nil
}
func (mux *kvMux) NumReplicas() int {
clientMux := mux.getState()
if clientMux == nil {
return 0
}
if clientMux.VBMap() == nil {
return 0
}
return clientMux.VBMap().NumReplicas()
}
func (mux *kvMux) BucketType() bucketType {
clientMux := mux.getState()
if clientMux == nil {
return bktTypeInvalid
}
return clientMux.BucketType()
}
func (mux *kvMux) SupportsGCCCP() bool {
clientMux := mux.getState()
if clientMux == nil {
return false
}
return clientMux.BucketType() == bktTypeNone
}
func (mux *kvMux) NumPipelines() int {
clientMux := mux.getState()
if clientMux == nil {
return 0
}
return clientMux.NumPipelines()
}
// CollectionsEnaled returns whether or not the kv mux was created with collections enabled.
func (mux *kvMux) CollectionsEnabled() bool {
return mux.collectionsEnabled
}
func (mux *kvMux) IsSecure() bool {
return mux.getState().tlsConfig != nil
}
// SupportsCollections returns whether or not collections are enabled AND supported by the server.
func (mux *kvMux) SupportsCollections() bool {
if !mux.collectionsEnabled {
return false
}
clientMux := mux.getState()
if clientMux == nil {
return false
}
return clientMux.collectionsSupported
}
func (mux *kvMux) HasBucketCapabilityStatus(cap BucketCapability, status BucketCapabilityStatus) bool {
clientMux := mux.getState()
if clientMux == nil {
return status == BucketCapabilityStatusUnknown
}
return clientMux.HasBucketCapabilityStatus(cap, status)
}
func (mux *kvMux) BucketCapabilityStatus(cap BucketCapability) BucketCapabilityStatus {
clientMux := mux.getState()
if clientMux == nil || clientMux.RevID() == -1 {
return BucketCapabilityStatusUnknown
}
return clientMux.BucketCapabilityStatus(cap)
}
func (mux *kvMux) RouteRequest(req *memdQRequest) (*memdPipeline, error) {
clientMux := mux.getState()
if clientMux == nil {
return nil, errShutdown
}
// We haven't seen a valid config yet so put this in the dead pipeline so
// it'll get requeued once we do get a config.
if clientMux.RevID() == -1 {
return clientMux.deadPipe, nil
}
var srvIdx int
repIdx := req.ReplicaIdx
// Route to specific server
if repIdx < 0 {
srvIdx = -repIdx - 1
} else {
var err error
bktType := clientMux.BucketType()
if bktType == bktTypeCouchbase {
if req.Key != nil {
req.Vbucket = clientMux.VBMap().VbucketByKey(req.Key)
}
srvIdx, err = clientMux.VBMap().NodeByVbucket(req.Vbucket, uint32(repIdx))
if err != nil {
return nil, err
}
} else if bktType == bktTypeMemcached {
if repIdx > 0 {
// Error. Memcached buckets don't understand replicas!
return nil, errInvalidReplica
}
if len(req.Key) == 0 {
// Non-broadcast keyless Memcached bucket request
return nil, errInvalidArgument
}
srvIdx, err = clientMux.KetamaMap().NodeByKey(req.Key)
if err != nil {
return nil, err
}
} else if bktType == bktTypeNone {
// This means that we're using GCCCP and not connected to a bucket
return nil, errGCCCPInUse
}
}
return clientMux.GetPipeline(srvIdx), nil
}
func (mux *kvMux) DispatchDirect(req *memdQRequest) (PendingOp, error) {
mux.tracer.StartCmdTrace(req)
req.dispatchTime = time.Now()
for {
pipeline, err := mux.RouteRequest(req)
if err != nil {
return nil, err
}
err = pipeline.SendRequest(req)
if err == errPipelineClosed {
continue
} else if err != nil {
if err == errPipelineFull {
err = errOverload
}
shortCircuit, routeErr := mux.handleOpRoutingResp(nil, req, err)
if shortCircuit {
return req, nil
}
return nil, routeErr
}
break
}
return req, nil
}
func (mux *kvMux) RequeueDirect(req *memdQRequest, isRetry bool) {
mux.tracer.StartCmdTrace(req)
handleError := func(err error) {
// We only want to log an error on retries if the error isn't cancelled.
if !isRetry || (isRetry && !errors.Is(err, ErrRequestCanceled)) {
logErrorf("Reschedule failed, failing request, Opaque=%d, Opcode=0x%x, (%s)", req.Opaque, req.Command, err)
}
req.tryCallback(nil, err)
}
logDebugf("Request being requeued, Opaque=%d, Opcode=0x%x", req.Opaque, req.Command)
for {
pipeline, err := mux.RouteRequest(req)
if err != nil {
handleError(err)
return
}
err = pipeline.RequeueRequest(req)
if err == errPipelineClosed {
continue
} else if err != nil {
handleError(err)
return
}
break
}
}
func (mux *kvMux) DispatchDirectToAddress(req *memdQRequest, pipeline *memdPipeline) (PendingOp, error) {
mux.tracer.StartCmdTrace(req)
req.dispatchTime = time.Now()
// We set the ReplicaIdx to a negative number to ensure it is not redispatched
// and we check that it was 0 to begin with to ensure it wasn't miss-used.
if req.ReplicaIdx != 0 {
return nil, errInvalidReplica
}
req.ReplicaIdx = -999999999
for {
err := pipeline.SendRequest(req)
if err == errPipelineClosed {
continue
} else if err != nil {
if err == errPipelineFull {
err = errOverload
}
shortCircuit, routeErr := mux.handleOpRoutingResp(nil, req, err)
if shortCircuit {
return req, nil
}
return nil, routeErr
}
break
}
return req, nil
}
func (mux *kvMux) Close() error {
mux.cfgMgr.RemoveConfigWatcher(mux)
clientMux := mux.clear()
if clientMux == nil {
return errShutdown
}
// Trigger any memdclients that are in graceful close to forcibly close.
close(mux.shutdownSig)
var muxErr error
// Shut down the client multiplexer which will close all its queues
// effectively causing all the clients to shut down.
for _, pipeline := range clientMux.pipelines {
err := pipeline.Close()
if err != nil {
logErrorf("failed to shut down pipeline: %s", err)
muxErr = errCliInternalError
}
}
if clientMux.deadPipe != nil {
err := clientMux.deadPipe.Close()
if err != nil {
logErrorf("failed to shut down deadpipe: %s", err)
muxErr = errCliInternalError
}
}
// Drain all the pipelines and error their requests, then
// drain the dead queue and error those requests.
cb := func(req *memdQRequest) {
req.tryCallback(nil, errShutdown)
}
mux.drainPipelines(clientMux, cb)
mux.clientCloseWg.Wait()
return muxErr
}
func (mux *kvMux) ForceReconnect(tlsConfig *dynTLSConfig, authMechanisms []AuthMechanism, auth AuthProvider,
reconnectLocal bool) {
logDebugf("Forcing reconnect of all connections")
mux.muxStateWriteLock.Lock()
muxState := mux.getState()
newMuxState := mux.newKVMuxState(muxState.RouteConfig(), tlsConfig, authMechanisms, auth)
atomic.SwapPointer(&mux.muxPtr, unsafe.Pointer(newMuxState))
mux.reconnectPipelines(muxState, newMuxState, reconnectLocal)
mux.muxStateWriteLock.Unlock()
}
func (mux *kvMux) PipelineSnapshot() (*pipelineSnapshot, error) {
clientMux := mux.getState()
if clientMux == nil {
return nil, errShutdown
}
return &pipelineSnapshot{
state: clientMux,
}, nil
}
type waitForConfigSnapshotOp struct {
cancelCh chan struct{}
}
func (w *waitForConfigSnapshotOp) Cancel() {
close(w.cancelCh)
}
func (mux *kvMux) WaitForConfigSnapshot(deadline time.Time, cb WaitForConfigSnapshotCallback) (PendingOp, error) {
// No point in doing anything if we're shutdown.
clientMux := mux.getState()
if clientMux == nil {
return nil, errShutdown
}
op := &waitForConfigSnapshotOp{
cancelCh: make(chan struct{}),
}
start := time.Now()
go func() {
select {
case <-mux.shutdownSig:
cb(nil, errShutdown)
case <-op.cancelCh:
cb(nil, errRequestCanceled)
case <-time.After(time.Until(deadline)):
cb(nil, &TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "WaitForConfigSnapshot",
TimeObserved: time.Since(start),
})
case <-mux.hasSeenConfigCh:
// Just in case.
clientMux := mux.getState()
if clientMux == nil {
cb(nil, errShutdown)
return
}
cb(&WaitForConfigSnapshotResult{
Snapshot: &ConfigSnapshot{
state: clientMux,
},
}, nil)
}
}()
return op, nil
}
func (mux *kvMux) ConfigSnapshot() (*ConfigSnapshot, error) {
clientMux := mux.getState()
if clientMux == nil {
return nil, errShutdown
}
return &ConfigSnapshot{
state: clientMux,
}, nil
}
func (mux *kvMux) handleOpRoutingResp(resp *memdQResponse, req *memdQRequest, originalErr error) (bool, error) {
// If there is no error, we should return immediately
if originalErr == nil {
return false, nil
}
// If this operation has been cancelled, we just fail immediately.
if errors.Is(originalErr, ErrRequestCanceled) || errors.Is(originalErr, ErrTimeout) {
return false, originalErr
}
err := translateMemdError(originalErr, req)
if err == originalErr {
if errors.Is(err, io.EOF) && !mux.closed() {
// The connection has gone away.
if req.Command == memd.CmdGetClusterConfig {
return false, err
}
// If the request is idempotent or not written yet then we should retry.
if req.Idempotent() || req.ConnectionInfo().lastDispatchedTo == "" {
if mux.waitAndRetryOperation(req, SocketNotAvailableRetryReason) {
return true, nil
}
}
} else if errors.Is(err, ErrMemdClientClosed) && !mux.closed() {
if req.Command == memd.CmdGetClusterConfig {
return false, err
}
// The request can't have been dispatched yet.
if mux.waitAndRetryOperation(req, SocketNotAvailableRetryReason) {
return true, nil
}
} else if errors.Is(err, io.ErrShortWrite) {
// This is a special case where the write has failed on the underlying connection and not all the bytes
// were written to the network.
if mux.waitAndRetryOperation(req, MemdWriteFailure) {
return true, nil
}
} else if resp != nil && resp.Magic == memd.CmdMagicRes {
// We don't know anything about this error so send it to the error map
shouldRetry := mux.errMapMgr.ShouldRetry(resp.Status)
if shouldRetry {
if mux.waitAndRetryOperation(req, KVErrMapRetryReason) {
return true, nil
}
}
}
} else {
// Handle potentially retrying the operation
if errors.Is(err, ErrNotMyVBucket) {
if mux.handleNotMyVbucket(resp, req) {
return true, nil
}
} else if errors.Is(err, ErrDocumentLocked) {
if mux.waitAndRetryOperation(req, KVLockedRetryReason) {
return true, nil
}
} else if errors.Is(err, ErrTemporaryFailure) {
if mux.waitAndRetryOperation(req, KVTemporaryFailureRetryReason) {
return true, nil
}
} else if errors.Is(err, ErrDurableWriteInProgress) {
if mux.waitAndRetryOperation(req, KVSyncWriteInProgressRetryReason) {
return true, nil
}
} else if errors.Is(err, ErrDurableWriteReCommitInProgress) {
if mux.waitAndRetryOperation(req, KVSyncWriteRecommitInProgressRetryReason) {
return true, nil
}
}
// If an error isn't in this list then we know what this error is but we don't support retries for it.
}
err = mux.errMapMgr.EnhanceKvError(err, resp, req)
if mux.postCompleteErrHandler == nil {
return false, err
}
return mux.postCompleteErrHandler(resp, req, err)
}
func (mux *kvMux) closed() bool {
return mux.getState() == nil
}
func (mux *kvMux) waitAndRetryOperation(req *memdQRequest, reason RetryReason) bool {
shouldRetry, retryTime := retryOrchMaybeRetry(req, reason)
if shouldRetry {
go func() {
time.Sleep(time.Until(retryTime))
mux.RequeueDirect(req, true)
}()
return true
}
return false
}
func (mux *kvMux) handleNotMyVbucket(resp *memdQResponse, req *memdQRequest) bool {
// Grab just the hostname from the source address
sourceHost, err := hostFromHostPort(resp.sourceAddr)
if err != nil {
logErrorf("NMV response source address was invalid, skipping config update")
} else {
// Try to parse the value as a bucket configuration
bk, err := parseConfig(resp.Value, sourceHost)
if err == nil {
// We need to push this upstream which will then update us with a new config.
mux.cfgMgr.OnNewConfig(bk)
}
}
if req.Command == memd.CmdRangeScanContinue {
// For range scan continue we never want to retry, the range scan is now invalid.
return false
}
// Redirect it! This may actually come back to this server, but I won't tell
// if you don't ;)
return mux.waitAndRetryOperation(req, KVNotMyVBucketRetryReason)
}
func (mux *kvMux) drainPipelines(clientMux *kvMuxState, cb func(req *memdQRequest)) {
for _, pipeline := range clientMux.pipelines {
logDebugf("Draining queue %+v", pipeline)
pipeline.Drain(cb)
}
if clientMux.deadPipe != nil {
clientMux.deadPipe.Drain(cb)
}
}
func (mux *kvMux) newKVMuxState(cfg *routeConfig, tlsConfig *dynTLSConfig, authMechanisms []AuthMechanism,
auth AuthProvider) *kvMuxState {
poolSize := 1
if !cfg.IsGCCCPConfig() {
poolSize = mux.poolSize
}
useTls := tlsConfig != nil
var kvServerList []routeEndpoint
if mux.noTLSSeedNode {
// The order of the kv server list matters, so we need to maintain the same order and just replace the seed
// node.
if useTls {
kvServerList = make([]routeEndpoint, len(cfg.kvServerList.SSLEndpoints))
copy(kvServerList, cfg.kvServerList.SSLEndpoints)
for i, ep := range cfg.kvServerList.NonSSLEndpoints {
if ep.IsSeedNode {
kvServerList[i] = ep
}
}
} else {
kvServerList = cfg.kvServerList.NonSSLEndpoints
}
} else {
if useTls {
kvServerList = cfg.kvServerList.SSLEndpoints
} else {
kvServerList = cfg.kvServerList.NonSSLEndpoints
}
}
var buffer bytes.Buffer
buffer.WriteString(fmt.Sprintln("KV muxer applying endpoints:"))
buffer.WriteString(fmt.Sprintf("Bucket: %s\n", cfg.name))
for _, ep := range kvServerList {
buffer.WriteString(fmt.Sprintf(" - %s\n", ep.Address))
}
logDebugf(buffer.String())
pipelines := make([]*memdPipeline, len(kvServerList))
for i, hostPort := range kvServerList {
trimmedHostPort := routeEndpoint{
Address: trimSchemePrefix(hostPort.Address),
IsSeedNode: hostPort.IsSeedNode,
}
getCurClientFn := func(cancelSig <-chan struct{}) (*memdClient, error) {
return mux.dialer.SlowDialMemdClient(cancelSig, trimmedHostPort, tlsConfig, auth, authMechanisms, mux.handleOpRoutingResp)
}
pipeline := newPipeline(trimmedHostPort, poolSize, mux.queueSize, getCurClientFn)
pipelines[i] = pipeline
}
return newKVMuxState(cfg, kvServerList, tlsConfig, authMechanisms, auth, pipelines,
newDeadPipeline(mux.queueSize))
}
func (mux *kvMux) reconnectPipelines(oldMuxState *kvMuxState, newMuxState *kvMuxState, reconnectSeed bool) {
oldPipelines := list.New()
for _, pipeline := range oldMuxState.pipelines {
oldPipelines.PushBack(pipeline)
}
for _, pipeline := range newMuxState.pipelines {
// If we aren't reconnecting the seed node then we need to take its clients and make sure we don't
// end up closing it down.
if pipeline.isSeedNode && !reconnectSeed {
oldPipeline := mux.stealPipeline(pipeline.Address(), oldPipelines)
if oldPipeline != nil {
pipeline.Takeover(oldPipeline)
}
}
pipeline.StartClients()
}
for e := oldPipelines.Front(); e != nil; e = e.Next() {
pipeline, ok := e.Value.(*memdPipeline)
if !ok {
logErrorf("Failed to cast old pipeline")
continue
}
clients := pipeline.GracefulClose()
for _, client := range clients {
mux.closeMemdClient(client, errForcedReconnect)
}
}
}
func (mux *kvMux) requeueRequests(oldMuxState *kvMuxState) {
// Gather all the requests from all the old pipelines and then
// sort and redispatch them (which will use the new pipelines)
var requestList []*memdQRequest
mux.drainPipelines(oldMuxState, func(req *memdQRequest) {
requestList = append(requestList, req)
})
sort.Sort(memdQRequestSorter(requestList))
for _, req := range requestList {
stopCmdTrace(req)
// If the command is a get cluster config then we cancel it rather than requeuing.
// Get cluster config is explicitly sent a specific pipeline so we do not want to requeue.
// This may seem like it'll cause the poller to take longer to fetch a config but that's
// OK because we can only have here by something fetching a new config anyway.
if req.Command == memd.CmdGetClusterConfig {
req.tryCallback(nil, ErrRequestCanceled)
continue
}
mux.RequeueDirect(req, false)
}
}
// closeMemdClient will gracefully close the memdclient, spinning up a goroutine to watch for when the client
// shuts down. The error provided is the error sent to any callback handlers for persistent operations which are
// currently live in the client.
func (mux *kvMux) closeMemdClient(client *memdClient, err error) {
mux.clientCloseWg.Add(1)
client.GracefulClose(err)
go func(client *memdClient) {
select {
case <-client.CloseNotify():
logDebugf("Memdclient %s/%p completed graceful shutdown", client.Address(), client)
case <-mux.shutdownSig:
logDebugf("Memdclient %s/%p being forcibly shutdown", client.Address(), client)
// Force the client to close even if there are requests in flight.
err := client.Close()
if err != nil {
logErrorf("failed to shutdown memdclient: %s", err)
}
<-client.CloseNotify()
logDebugf("Memdclient %s/%p completed shutdown", client.Address(), client)
}
mux.clientCloseWg.Done()
}(client)
}
func (mux *kvMux) stealPipeline(address string, oldPipelines *list.List) *memdPipeline {
for e := oldPipelines.Front(); e != nil; e = e.Next() {
pipeline, ok := e.Value.(*memdPipeline)
if !ok {
logErrorf("Failed to cast old pipeline")
continue
}
if pipeline.Address() == address {
oldPipelines.Remove(e)
return pipeline
}
}
return nil
}
func (mux *kvMux) pipelineTakeover(oldMux, newMux *kvMuxState) {
oldPipelines := list.New()
// Gather all our old pipelines up for takeover and what not
if oldMux != nil {
for _, pipeline := range oldMux.pipelines {
oldPipelines.PushBack(pipeline)
}
}
// Initialize new pipelines (possibly with a takeover)
for _, pipeline := range newMux.pipelines {
oldPipeline := mux.stealPipeline(pipeline.Address(), oldPipelines)
if oldPipeline != nil {
pipeline.Takeover(oldPipeline)
}
pipeline.StartClients()
}
// Shut down any pipelines that were not taken over
for e := oldPipelines.Front(); e != nil; e = e.Next() {
pipeline, ok := e.Value.(*memdPipeline)
if !ok {
logErrorf("Failed to cast old pipeline")
continue
}
clients := pipeline.GracefulClose()
for _, client := range clients {
mux.closeMemdClient(client, nil)
}
}
if oldMux != nil && oldMux.deadPipe != nil {
err := oldMux.deadPipe.Close()
if err != nil {
logErrorf("Failed to properly close abandoned dead pipe (%s)", err)
}
}
}
gocbcore-10.2.3/kvmux_test.go 0000664 0000000 0000000 00000007420 14417540156 0016132 0 ustar 00root root 0000000 0000000 package gocbcore
func (suite *StandardTestSuite) TestKvMux_HasBucketCapabilityStatusNoState() {
// No mux state, shouldn't actually happen in practise.
mux := kvMux{}
suite.Assert().True(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusUnknown))
suite.Assert().False(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusSupported))
suite.Assert().False(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusUnsupported))
suite.Assert().True(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusUnknown))
suite.Assert().False(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusSupported))
suite.Assert().False(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusUnsupported))
}
func (suite *StandardTestSuite) TestKvMux_HasBucketCapabilityStatusBlankState() {
cfg := &routeConfig{
revID: -1,
}
// Mux state as if we haven't received a config yet.
muxState := newKVMuxState(cfg, nil, nil, nil, nil, nil, nil)
mux := kvMux{}
mux.updateState(nil, muxState)
suite.Assert().True(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusUnknown))
suite.Assert().False(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusSupported))
suite.Assert().False(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusUnsupported))
suite.Assert().False(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusUnknown))
suite.Assert().False(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusSupported))
suite.Assert().True(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusUnsupported))
}
func (suite *StandardTestSuite) TestKvMux_HasBucketCapabilityStatusUnsupported() {
// Mux state as if we have received a config yet.
muxState := &kvMuxState{
routeCfg: routeConfig{
revID: 1,
},
bucketCapabilities: map[BucketCapability]BucketCapabilityStatus{
BucketCapabilityReplaceBodyWithXattr: BucketCapabilityStatusUnsupported,
},
}
mux := kvMux{}
mux.updateState(nil, muxState)
suite.Assert().False(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusUnknown))
suite.Assert().False(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusSupported))
suite.Assert().True(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusUnsupported))
suite.Assert().False(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusUnknown))
suite.Assert().False(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusSupported))
suite.Assert().True(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusUnsupported))
}
func (suite *StandardTestSuite) TestKvMux_HasBucketCapabilityStatusSupported() {
// Mux state as if we have received a config yet.
muxState := &kvMuxState{
routeCfg: routeConfig{
revID: 1,
},
bucketCapabilities: map[BucketCapability]BucketCapabilityStatus{
BucketCapabilityReplaceBodyWithXattr: BucketCapabilityStatusSupported,
},
}
mux := kvMux{}
mux.updateState(nil, muxState)
suite.Assert().False(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusUnknown))
suite.Assert().True(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusSupported))
suite.Assert().False(mux.HasBucketCapabilityStatus(BucketCapabilityReplaceBodyWithXattr, BucketCapabilityStatusUnsupported))
suite.Assert().False(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusUnknown))
suite.Assert().False(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusSupported))
suite.Assert().True(mux.HasBucketCapabilityStatus(9999, BucketCapabilityStatusUnsupported))
}
gocbcore-10.2.3/kvmuxstate.go 0000664 0000000 0000000 00000007257 14417540156 0016144 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"fmt"
)
type kvMuxState struct {
pipelines []*memdPipeline
deadPipe *memdPipeline
routeCfg routeConfig
bucketCapabilities map[BucketCapability]BucketCapabilityStatus
collectionsSupported bool
kvServerList []routeEndpoint
tlsConfig *dynTLSConfig
authMechanisms []AuthMechanism
auth AuthProvider
}
func newKVMuxState(cfg *routeConfig, kvServerList []routeEndpoint, tlsConfig *dynTLSConfig,
authMechanisms []AuthMechanism, auth AuthProvider, pipelines []*memdPipeline, deadpipe *memdPipeline) *kvMuxState {
mux := &kvMuxState{
pipelines: pipelines,
deadPipe: deadpipe,
routeCfg: *cfg,
bucketCapabilities: map[BucketCapability]BucketCapabilityStatus{
BucketCapabilityDurableWrites: BucketCapabilityStatusUnknown,
BucketCapabilityCreateAsDeleted: BucketCapabilityStatusUnknown,
BucketCapabilityReplaceBodyWithXattr: BucketCapabilityStatusUnknown,
},
collectionsSupported: cfg.ContainsBucketCapability("collections"),
kvServerList: kvServerList,
tlsConfig: tlsConfig,
authMechanisms: authMechanisms,
auth: auth,
}
// We setup with a fake config, this means that durability support is still unknown.
if cfg.revID > -1 {
if cfg.ContainsBucketCapability("durableWrite") {
mux.bucketCapabilities[BucketCapabilityDurableWrites] = BucketCapabilityStatusSupported
} else {
mux.bucketCapabilities[BucketCapabilityDurableWrites] = BucketCapabilityStatusUnsupported
}
if cfg.ContainsBucketCapability("tombstonedUserXAttrs") {
mux.bucketCapabilities[BucketCapabilityCreateAsDeleted] = BucketCapabilityStatusSupported
} else {
mux.bucketCapabilities[BucketCapabilityCreateAsDeleted] = BucketCapabilityStatusUnsupported
}
if cfg.ContainsBucketCapability("subdoc.ReplaceBodyWithXattr") {
mux.bucketCapabilities[BucketCapabilityReplaceBodyWithXattr] = BucketCapabilityStatusSupported
} else {
mux.bucketCapabilities[BucketCapabilityReplaceBodyWithXattr] = BucketCapabilityStatusUnsupported
}
}
return mux
}
func (mux *kvMuxState) RouteConfig() *routeConfig {
return &mux.routeCfg
}
func (mux *kvMuxState) RevID() int64 {
return mux.routeCfg.revID
}
func (mux *kvMuxState) VBMap() *vbucketMap {
return mux.routeCfg.vbMap
}
func (mux *kvMuxState) UUID() string {
return mux.routeCfg.uuid
}
func (mux *kvMuxState) KetamaMap() *ketamaContinuum {
return mux.routeCfg.ketamaMap
}
func (mux *kvMuxState) BucketType() bucketType {
return mux.routeCfg.bktType
}
func (mux *kvMuxState) KVEps() []string {
var epList []string
for _, s := range mux.kvServerList {
epList = append(epList, s.Address)
}
return epList
}
func (mux *kvMuxState) NumPipelines() int {
return len(mux.pipelines)
}
func (mux *kvMuxState) GetPipeline(index int) *memdPipeline {
if index < 0 || index >= len(mux.pipelines) {
return mux.deadPipe
}
return mux.pipelines[index]
}
func (mux *kvMuxState) HasBucketCapabilityStatus(cap BucketCapability, status BucketCapabilityStatus) bool {
st, ok := mux.bucketCapabilities[cap]
if !ok {
return status == BucketCapabilityStatusUnsupported
}
return st == status
}
func (mux *kvMuxState) BucketCapabilityStatus(cap BucketCapability) BucketCapabilityStatus {
st, ok := mux.bucketCapabilities[cap]
if !ok {
return BucketCapabilityStatusUnsupported
}
return st
}
// nolint: unused
func (mux *kvMuxState) debugString() string {
var outStr string
for i, n := range mux.pipelines {
outStr += fmt.Sprintf("Pipeline %d:\n", i)
outStr += reindentLog(" ", n.debugString()) + "\n"
}
outStr += "Dead Pipeline:\n"
if mux.deadPipe != nil {
outStr += reindentLog(" ", mux.deadPipe.debugString()) + "\n"
} else {
outStr += " Disabled\n"
}
return outStr
}
gocbcore-10.2.3/kvmuxstate_test.go 0000664 0000000 0000000 00000002065 14417540156 0017173 0 ustar 00root root 0000000 0000000 package gocbcore
func (suite *StandardTestSuite) TestKvMuxState_BucketCapabilities_InitialConfig() {
cfg := &routeConfig{
revID: -1,
}
muxState := newKVMuxState(cfg, nil, nil, nil, nil, nil, nil)
suite.Assert().Equal(map[BucketCapability]BucketCapabilityStatus{
BucketCapabilityDurableWrites: BucketCapabilityStatusUnknown,
BucketCapabilityCreateAsDeleted: BucketCapabilityStatusUnknown,
BucketCapabilityReplaceBodyWithXattr: BucketCapabilityStatusUnknown,
}, muxState.bucketCapabilities)
}
func (suite *StandardTestSuite) TestKvMuxState_BucketCapabilities() {
cfg := &routeConfig{
revID: 1,
bucketCapabilities: []string{"durableWrite"},
}
muxState := newKVMuxState(cfg, nil, nil, nil, nil, nil, nil)
suite.Assert().Equal(map[BucketCapability]BucketCapabilityStatus{
BucketCapabilityDurableWrites: BucketCapabilityStatusSupported,
BucketCapabilityCreateAsDeleted: BucketCapabilityStatusUnsupported,
BucketCapabilityReplaceBodyWithXattr: BucketCapabilityStatusUnsupported,
}, muxState.bucketCapabilities)
}
gocbcore-10.2.3/logging.go 0000664 0000000 0000000 00000011225 14417540156 0015345 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"fmt"
"log"
"os"
"strings"
)
// LogLevel specifies the severity of a log message.
type LogLevel int
// Various logging levels (or subsystems) which can categorize the message.
// Currently these are ordered in decreasing severity.
const (
LogError LogLevel = iota
LogWarn
LogInfo
LogDebug
LogTrace
LogSched
LogMaxVerbosity
)
func redactUserData(v interface{}) string {
return fmt.Sprintf("%v", v)
}
func redactMetaData(v interface{}) string {
return fmt.Sprintf("%v", v)
}
func redactSystemData(v interface{}) string {
return fmt.Sprintf("%v", v)
}
// LogRedactLevel specifies the degree with which to redact the logs.
type LogRedactLevel int
const (
// RedactNone indicates to perform no redactions
RedactNone LogRedactLevel = iota
// RedactPartial indicates to redact all possible user-identifying information from logs.
RedactPartial
// RedactFull indicates to fully redact all possible identifying information from logs.
RedactFull
)
// SetLogRedactionLevel specifies the level with which logs should be redacted.
func SetLogRedactionLevel(level LogRedactLevel) {
globalLogRedactionLevel = level
}
func isLogRedactionLevelNone() bool {
return globalLogRedactionLevel == RedactNone
}
func isLogRedactionLevelPartial() bool {
return globalLogRedactionLevel == RedactPartial
}
func isLogRedactionLevelFull() bool {
return globalLogRedactionLevel == RedactFull
}
func logLevelToString(level LogLevel) string {
switch level {
case LogError:
return "error"
case LogWarn:
return "warn"
case LogInfo:
return "info"
case LogDebug:
return "debug"
case LogTrace:
return "trace"
case LogSched:
return "sched"
}
return fmt.Sprintf("unknown (%d)", level)
}
// Logger defines a logging interface. You can either use one of the default loggers
// (DefaultStdioLogger(), VerboseStdioLogger()) or implement your own.
type Logger interface {
// Outputs logging information:
// level is the verbosity level
// offset is the position within the calling stack from which the message
// originated. This is useful for contextual loggers which retrieve file/line
// information.
Log(level LogLevel, offset int, format string, v ...interface{}) error
}
type defaultLogger struct {
Level LogLevel
GoLogger *log.Logger
}
func (l *defaultLogger) Log(level LogLevel, offset int, format string, v ...interface{}) error {
if level > l.Level {
return nil
}
s := fmt.Sprintf(format, v...)
return l.GoLogger.Output(offset+2, s)
}
var (
globalDefaultLogger = defaultLogger{
GoLogger: log.New(os.Stderr, "GOCB ", log.Lmicroseconds|log.Lshortfile), Level: LogDebug,
}
globalVerboseLogger = defaultLogger{
GoLogger: globalDefaultLogger.GoLogger, Level: LogMaxVerbosity,
}
globalLogger Logger
globalLogRedactionLevel LogRedactLevel
)
// DefaultStdioLogger gets the default standard I/O logger.
// gocbcore.SetLogger(gocbcore.DefaultStdioLogger())
func DefaultStdioLogger() Logger {
return &globalDefaultLogger
}
// VerboseStdioLogger is a more verbose level of DefaultStdioLogger(). Messages
// pertaining to the scheduling of ordinary commands (and their responses) will
// also be emitted.
// gocbcore.SetLogger(gocbcore.VerboseStdioLogger())
func VerboseStdioLogger() Logger {
return &globalVerboseLogger
}
// SetLogger sets a logger to be used by the library. A logger can be obtained via
// the DefaultStdioLogger() or VerboseStdioLogger() functions. You can also implement
// your own logger using the Logger interface.
func SetLogger(logger Logger) {
globalLogger = logger
}
type redactableLogValue interface {
redacted() interface{}
}
func logExf(level LogLevel, offset int, format string, v ...interface{}) {
if globalLogger != nil {
if level <= LogInfo && !isLogRedactionLevelNone() {
// We only redact at info level or below.
for i, iv := range v {
if redactable, ok := iv.(redactableLogValue); ok {
v[i] = redactable.redacted()
}
}
}
err := globalLogger.Log(level, offset+1, format, v...)
if err != nil {
log.Printf("Logger error occurred (%s)\n", err)
}
}
}
func logDebugf(format string, v ...interface{}) {
logExf(LogDebug, 1, format, v...)
}
func logSchedf(format string, v ...interface{}) {
logExf(LogSched, 1, format, v...)
}
func logWarnf(format string, v ...interface{}) {
logExf(LogWarn, 1, format, v...)
}
func logErrorf(format string, v ...interface{}) {
logExf(LogError, 1, format, v...)
}
func logInfof(format string, v ...interface{}) {
logExf(LogInfo, 1, format, v...)
}
func reindentLog(indent, message string) string {
reindentedMessage := strings.Replace(message, "\n", "\n"+indent, -1)
return fmt.Sprintf("%s%s", indent, reindentedMessage)
}
gocbcore-10.2.3/logging_test.go 0000664 0000000 0000000 00000001417 14417540156 0016406 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"bytes"
"log"
)
func (suite *UnitTestSuite) TestLogRedaction() {
var logs bytes.Buffer
gologger := log.New(&logs, "", 0)
sdklogger := defaultLogger{
GoLogger: gologger,
Level: LogDebug,
}
if suite.Assert().NoError(sdklogger.Log(LogDebug, 1, redactUserData("sensitive user data"))) {
suite.Assert().Equal("sensitive user data\n", logs.String())
}
logs.Reset()
if suite.Assert().NoError(sdklogger.Log(LogDebug, 1, redactMetaData("sensitive meta data"))) {
suite.Assert().Equal("sensitive meta data\n", logs.String())
}
logs.Reset()
if suite.Assert().NoError(sdklogger.Log(LogDebug, 1, redactSystemData("sensitive system data"))) {
suite.Assert().Equal("sensitive system data\n", logs.String())
}
}
gocbcore-10.2.3/memd/ 0000775 0000000 0000000 00000000000 14417540156 0014311 5 ustar 00root root 0000000 0000000 gocbcore-10.2.3/memd/README.md 0000664 0000000 0000000 00000000060 14417540156 0015564 0 ustar 00root root 0000000 0000000 This memd library should be moved into gocbcore! gocbcore-10.2.3/memd/cidsupporttable.go 0000664 0000000 0000000 00000002446 14417540156 0020052 0 ustar 00root root 0000000 0000000 package memd
var cidSupportedOps = []CmdCode{
CmdGet,
CmdSet,
CmdAdd,
CmdReplace,
CmdDelete,
CmdIncrement,
CmdDecrement,
CmdAppend,
CmdPrepend,
CmdTouch,
CmdGAT,
CmdGetReplica,
CmdGetLocked,
CmdUnlockKey,
CmdGetMeta,
CmdSetMeta,
CmdDelMeta,
CmdSubDocGet,
CmdSubDocExists,
CmdSubDocDictAdd,
CmdSubDocDictSet,
CmdSubDocDelete,
CmdSubDocReplace,
CmdSubDocArrayPushLast,
CmdSubDocArrayPushFirst,
CmdSubDocArrayInsert,
CmdSubDocArrayAddUnique,
CmdSubDocCounter,
CmdSubDocMultiLookup,
CmdSubDocMultiMutation,
CmdSubDocGetCount,
CmdDcpMutation,
CmdDcpExpiration,
CmdDcpDeletion,
}
func makeCidSupportedTable() []bool {
var cidTableLen uint32
for _, cmd := range cidSupportedOps {
if uint32(cmd) >= cidTableLen {
cidTableLen = uint32(cmd) + 1
}
}
cidTable := make([]bool, cidTableLen)
for _, cmd := range cidSupportedOps {
cidTable[cmd] = true
}
return cidTable
}
var cidSupportedTable = makeCidSupportedTable()
// IsCommandCollectionEncoded returns whether a particular command code
// should have its key collection encoded when collections support is
// enabled for a particular connection
func IsCommandCollectionEncoded(cmd CmdCode) bool {
cmdIdx := int(cmd)
if cmdIdx < 0 || cmdIdx >= len(cidSupportedTable) {
return false
}
return cidSupportedTable[cmdIdx]
}
gocbcore-10.2.3/memd/cmdcode.go 0000664 0000000 0000000 00000015532 14417540156 0016244 0 ustar 00root root 0000000 0000000 package memd
import (
"encoding/hex"
"fmt"
)
// CmdCode represents the specific command the packet is performing.
type CmdCode uint8
// These constants provide predefined values for all the operations
// which are supported by this library.
const (
CmdGet = CmdCode(0x00)
CmdSet = CmdCode(0x01)
CmdAdd = CmdCode(0x02)
CmdReplace = CmdCode(0x03)
CmdDelete = CmdCode(0x04)
CmdIncrement = CmdCode(0x05)
CmdDecrement = CmdCode(0x06)
CmdNoop = CmdCode(0x0a)
CmdAppend = CmdCode(0x0e)
CmdPrepend = CmdCode(0x0f)
CmdStat = CmdCode(0x10)
CmdTouch = CmdCode(0x1c)
CmdGAT = CmdCode(0x1d)
CmdHello = CmdCode(0x1f)
CmdSASLListMechs = CmdCode(0x20)
CmdSASLAuth = CmdCode(0x21)
CmdSASLStep = CmdCode(0x22)
CmdGetAllVBSeqnos = CmdCode(0x48)
CmdDcpOpenConnection = CmdCode(0x50)
CmdDcpAddStream = CmdCode(0x51)
CmdDcpCloseStream = CmdCode(0x52)
CmdDcpStreamReq = CmdCode(0x53)
CmdDcpGetFailoverLog = CmdCode(0x54)
CmdDcpStreamEnd = CmdCode(0x55)
CmdDcpSnapshotMarker = CmdCode(0x56)
CmdDcpMutation = CmdCode(0x57)
CmdDcpDeletion = CmdCode(0x58)
CmdDcpExpiration = CmdCode(0x59)
CmdDcpSeqNoAdvanced = CmdCode(0x64)
CmdDcpOsoSnapshot = CmdCode(0x65)
CmdDcpFlush = CmdCode(0x5a)
CmdDcpSetVbucketState = CmdCode(0x5b)
CmdDcpNoop = CmdCode(0x5c)
CmdDcpBufferAck = CmdCode(0x5d)
CmdDcpControl = CmdCode(0x5e)
CmdDcpEvent = CmdCode(0x5f)
CmdGetReplica = CmdCode(0x83)
CmdSelectBucket = CmdCode(0x89)
CmdObserveSeqNo = CmdCode(0x91)
CmdObserve = CmdCode(0x92)
CmdGetLocked = CmdCode(0x94)
CmdUnlockKey = CmdCode(0x95)
CmdGetMeta = CmdCode(0xa0)
CmdSetMeta = CmdCode(0xa2)
CmdDelMeta = CmdCode(0xa8)
CmdGetClusterConfig = CmdCode(0xb5)
CmdGetRandom = CmdCode(0xb6)
CmdCollectionsGetManifest = CmdCode(0xba)
CmdCollectionsGetID = CmdCode(0xbb)
CmdSubDocGet = CmdCode(0xc5)
CmdSubDocExists = CmdCode(0xc6)
CmdSubDocDictAdd = CmdCode(0xc7)
CmdSubDocDictSet = CmdCode(0xc8)
CmdSubDocDelete = CmdCode(0xc9)
CmdSubDocReplace = CmdCode(0xca)
CmdSubDocArrayPushLast = CmdCode(0xcb)
CmdSubDocArrayPushFirst = CmdCode(0xcc)
CmdSubDocArrayInsert = CmdCode(0xcd)
CmdSubDocArrayAddUnique = CmdCode(0xce)
CmdSubDocCounter = CmdCode(0xcf)
CmdSubDocMultiLookup = CmdCode(0xd0)
CmdSubDocMultiMutation = CmdCode(0xd1)
CmdSubDocGetCount = CmdCode(0xd2)
CmdSubDocReplaceBodyWithXattr = CmdCode(0xd3)
CmdRangeScanCreate = CmdCode(0xda)
CmdRangeScanContinue = CmdCode(0xdb)
CmdRangeScanCancel = CmdCode(0xdc)
CmdGetErrorMap = CmdCode(0xfe)
)
// Name returns the string representation of the CmdCode.
func (command CmdCode) Name() string {
switch command {
case CmdGet:
return "CMD_GET"
case CmdSet:
return "CMD_SET"
case CmdAdd:
return "CMD_ADD"
case CmdReplace:
return "CMD_REPLACE"
case CmdDelete:
return "CMD_DELETE"
case CmdIncrement:
return "CMD_INCREMENT"
case CmdDecrement:
return "CMD_DECREMENT"
case CmdNoop:
return "CMD_NOOP"
case CmdAppend:
return "CMD_APPEND"
case CmdPrepend:
return "CMD_PREPEND"
case CmdStat:
return "CMD_STAT"
case CmdTouch:
return "CMD_TOUCH"
case CmdGAT:
return "CMD_GAT"
case CmdHello:
return "CMD_HELLO"
case CmdSASLListMechs:
return "CMD_SASLLISTMECHS"
case CmdSASLAuth:
return "CMD_SASLAUTH"
case CmdSASLStep:
return "CMD_SASLSTEP"
case CmdGetAllVBSeqnos:
return "CMD_GETALLVBSEQNOS"
case CmdDcpOpenConnection:
return "CMD_DCPOPENCONNECTION"
case CmdDcpAddStream:
return "CMD_DCPADDSTREAM"
case CmdDcpCloseStream:
return "CMD_DCPCLOSESTREAM"
case CmdDcpStreamReq:
return "CMD_DCPSTREAMREQ"
case CmdDcpGetFailoverLog:
return "CMD_DCPGETFAILOVERLOG"
case CmdDcpStreamEnd:
return "CMD_DCPSTREAMEND"
case CmdDcpSnapshotMarker:
return "CMD_DCPSNAPSHOTMARKER"
case CmdDcpMutation:
return "CMD_DCPMUTATION"
case CmdDcpDeletion:
return "CMD_DCPDELETION"
case CmdDcpExpiration:
return "CMD_DCPEXPIRATION"
case CmdDcpFlush:
return "CMD_DCPFLUSH"
case CmdDcpSetVbucketState:
return "CMD_DCPSETVBUCKETSTATE"
case CmdDcpNoop:
return "CMD_DCPNOOP"
case CmdDcpBufferAck:
return "CMD_DCPBUFFERACK"
case CmdDcpControl:
return "CMD_DCPCONTROL"
case CmdGetReplica:
return "CMD_GETREPLICA"
case CmdSelectBucket:
return "CMD_SELECTBUCKET"
case CmdObserveSeqNo:
return "CMD_OBSERVESEQNO"
case CmdObserve:
return "CMD_OBSERVE"
case CmdGetLocked:
return "CMD_GETLOCKED"
case CmdUnlockKey:
return "CMD_UNLOCKKEY"
case CmdGetMeta:
return "CMD_GETMETA"
case CmdSetMeta:
return "CMD_SETMETA"
case CmdDelMeta:
return "CMD_DELMETA"
case CmdGetClusterConfig:
return "CMD_GETCLUSTERCONFIG"
case CmdGetRandom:
return "CMD_GETRANDOM"
case CmdSubDocGet:
return "CMD_SUBDOCGET"
case CmdSubDocExists:
return "CMD_SUBDOCEXISTS"
case CmdSubDocDictAdd:
return "CMD_SUBDOCDICTADD"
case CmdSubDocDictSet:
return "CMD_SUBDOCDICTSET"
case CmdSubDocDelete:
return "CMD_SUBDOCDELETE"
case CmdSubDocReplace:
return "CMD_SUBDOCREPLACE"
case CmdSubDocArrayPushLast:
return "CMD_SUBDOCARRAYPUSHLAST"
case CmdSubDocArrayPushFirst:
return "CMD_SUBDOCARRAYPUSHFIRST"
case CmdSubDocArrayInsert:
return "CMD_SUBDOCARRAYINSERT"
case CmdSubDocArrayAddUnique:
return "CMD_SUBDOCARRAYADDUNIQUE"
case CmdSubDocCounter:
return "CMD_SUBDOCCOUNTER"
case CmdSubDocMultiLookup:
return "CMD_SUBDOCMULTILOOKUP"
case CmdSubDocMultiMutation:
return "CMD_SUBDOCMULTIMUTATION"
case CmdSubDocGetCount:
return "CMD_SUBDOCGETCOUNT"
case CmdGetErrorMap:
return "CMD_GETERRORMAP"
case CmdCollectionsGetID:
return "CMD_GETCOLLECTIONID"
case CmdCollectionsGetManifest:
return "CMD_GETCOLLECTIONMANIFEST"
case CmdRangeScanCreate:
return "CMD_RANGESCANCREATE"
case CmdRangeScanContinue:
return "CMD_RANGESCANCONTINUE"
case CmdRangeScanCancel:
return "CMD_RANGESCANCANCEL"
default:
return "CMD_x" + hex.EncodeToString([]byte{byte(command)})
}
}
func (magic CmdMagic) String() string {
switch magic {
case CmdMagicReq:
return "CmdMagicReq"
case CmdMagicRes:
return "CmdMagicRes"
}
return fmt.Sprintf("CmdMagicUnk(%d)", magic)
}
gocbcore-10.2.3/memd/conn.go 0000664 0000000 0000000 00000040204 14417540156 0015575 0 ustar 00root root 0000000 0000000 package memd
import (
"bytes"
"encoding/binary"
"errors"
"io"
"sync"
"sync/atomic"
"time"
)
// writerBufPool - Thread safe pool containing packet write buffers i.e. they should be used to write a single packet to the
// TCP socket.
var writerBufPool = sync.Pool{
New: func() interface{} {
return bytes.NewBuffer(make([]byte, 0))
},
}
// aquireWriteBuf - Returns a pointer to a write buffer which is ready to be used, ensure the buffer is released using
// the 'releaseWriteBuf' function.
func aquireWriteBuf() *bytes.Buffer {
return writerBufPool.Get().(*bytes.Buffer)
}
// releaseWriteBuf - Reset the buffer so that it's clean for the next user (note that this retains the underlying
// storage for future writes) and then return it to the pool.
func releaseWriteBuf(buf *bytes.Buffer) {
buf.Reset()
writerBufPool.Put(buf)
}
// Conn represents a memcached protocol connection.
type Conn struct {
stream io.ReadWriter
headerBuf [24]byte
enabledFeatures uint64
}
// NewConn creates a new connection object which can be used to perform
// reading and writing of packets.
func NewConn(stream io.ReadWriter) *Conn {
return &Conn{
stream: stream,
}
}
// EnableFeature enables a particular feature on this connection.
func (c *Conn) EnableFeature(feature HelloFeature) {
featureBit := uint64(1) << int(feature)
for {
enabledFeatures := atomic.LoadUint64(&c.enabledFeatures)
if enabledFeatures&featureBit > 0 {
// already enabled
return
}
newEnabledFeatures := enabledFeatures | featureBit
if atomic.CompareAndSwapUint64(&c.enabledFeatures, enabledFeatures, newEnabledFeatures) {
break
}
}
}
// IsFeatureEnabled indicates whether a particular feature is enabled
// on this particular connection. Note that this is directly based on
// calls to EnableFeature and is not controlled by the library.
func (c *Conn) IsFeatureEnabled(feature HelloFeature) bool {
featureBit := uint64(1) << int(feature)
enabledFeatures := atomic.LoadUint64(&c.enabledFeatures)
return enabledFeatures&featureBit > 0
}
func (c *Conn) isCollectionsEnabled() bool {
return c.IsFeatureEnabled(FeatureCollections)
}
// WritePacket writes a packet to the network.
func (c *Conn) WritePacket(pkt *Packet) error {
encodedKey := pkt.Key
extras := pkt.Extras
if c.isCollectionsEnabled() {
if pkt.Command == CmdObserve {
// While it's possible that the Observe operation is in fact supported with collections
// enabled, we don't currently implement that operation for simplicity, as the key is
// actually hidden away in the value data instead of the usual key data.
return errors.New("the observe operation is not supported with collections enabled")
}
if IsCommandCollectionEncoded(pkt.Command) {
collEncodedKey := make([]byte, 0, len(encodedKey)+5)
collEncodedKey = AppendULEB128_32(collEncodedKey, pkt.CollectionID)
collEncodedKey = append(collEncodedKey, encodedKey...)
encodedKey = collEncodedKey
} else if pkt.Command == CmdGetRandom {
// GetRandom expects the cid to be in the extras
// GetRandom MUST not have any extras if not using collections so we're ok to just set it.
// It also doesn't expect the collection ID to be leb encoded.
extras = make([]byte, 4)
binary.BigEndian.PutUint32(extras, pkt.CollectionID)
} else {
if pkt.CollectionID > 0 {
return errors.New("cannot encode collection id with a non-collection command")
}
}
} else {
if pkt.CollectionID > 0 {
return errors.New("cannot encode collection id without the feature enabled")
}
}
extLen := len(extras)
keyLen := len(encodedKey)
valLen := len(pkt.Value)
framesLen := 0
if pkt.BarrierFrame != nil {
framesLen++
}
if pkt.DurabilityLevelFrame != nil {
if pkt.DurabilityTimeoutFrame == nil {
framesLen += 2
} else {
framesLen += 4
}
}
if pkt.StreamIDFrame != nil {
framesLen += 3
}
if pkt.OpenTracingFrame != nil {
framesLen += calcHeaderSize(len(pkt.OpenTracingFrame.TraceContext))
}
if pkt.ServerDurationFrame != nil {
framesLen += 3
}
if pkt.UserImpersonationFrame != nil {
framesLen += calcHeaderSize(len(pkt.UserImpersonationFrame.User))
}
if pkt.PreserveExpiryFrame != nil {
framesLen += 1
}
for _, fr := range pkt.UnsupportedFrames {
framesLen += calcHeaderSize(len(fr.Data))
}
// We automatically upgrade a packet from normal Req or Res magic into
// the frame variant depending on the usage of them.
pktMagic := pkt.Magic
if framesLen > 0 {
switch pktMagic {
case CmdMagicReq:
if !c.IsFeatureEnabled(FeatureAltRequests) {
return errors.New("cannot use frames in req packets without enabling the feature")
}
pktMagic = cmdMagicReqExt
case CmdMagicRes:
pktMagic = cmdMagicResExt
default:
return errors.New("cannot use frames with an unsupported magic")
}
}
buffer := aquireWriteBuf()
defer releaseWriteBuf(buffer)
buffer.WriteByte(byte(pktMagic))
buffer.WriteByte(byte(pkt.Command))
// This is safe to do without checking the magic as we check the magic
// above before incrementing the framesLen variable
if framesLen > 0 {
buffer.WriteByte(byte(framesLen))
buffer.WriteByte(byte(keyLen))
} else {
writeUint16(buffer, uint16(keyLen))
}
buffer.WriteByte(byte(extLen))
buffer.WriteByte(pkt.Datatype)
switch pkt.Magic {
case CmdMagicReq:
if pkt.Status != 0 {
return errors.New("cannot specify status in a request packet")
}
writeUint16(buffer, pkt.Vbucket)
case CmdMagicRes:
if pkt.Vbucket != 0 {
return errors.New("cannot specify vbucket in a response packet")
}
writeUint16(buffer, uint16(pkt.Status))
default:
return errors.New("cannot encode status/vbucket for unknown packet magic")
}
writeUint32(buffer, uint32(keyLen+extLen+valLen+framesLen))
writeUint32(buffer, pkt.Opaque)
writeUint64(buffer, pkt.Cas)
// Generate the framing extra data
if pkt.BarrierFrame != nil {
if pkt.Magic != CmdMagicReq {
return errors.New("cannot use barrier frame in non-request packets")
}
writeFrameHeader(buffer, frameTypeReqBarrier, 0)
}
if pkt.DurabilityLevelFrame != nil || pkt.DurabilityTimeoutFrame != nil {
if pkt.Magic != CmdMagicReq {
return errors.New("cannot use durability level frame in non-request packets")
}
if !c.IsFeatureEnabled(FeatureSyncReplication) {
return errors.New("cannot use sync replication frames without enabling the feature")
}
if pkt.DurabilityLevelFrame == nil && pkt.DurabilityTimeoutFrame != nil {
return errors.New("cannot encode durability timeout frame without durability level frame")
}
if pkt.DurabilityTimeoutFrame == nil {
writeFrameHeader(buffer, frameTypeReqSyncDurability, 1)
buffer.WriteByte(byte(pkt.DurabilityLevelFrame.DurabilityLevel))
} else {
durabilityTimeoutMillis := pkt.DurabilityTimeoutFrame.DurabilityTimeout / time.Millisecond
if durabilityTimeoutMillis > 65535 {
durabilityTimeoutMillis = 65535
}
writeFrameHeader(buffer, frameTypeReqSyncDurability, 3)
buffer.WriteByte(byte(pkt.DurabilityLevelFrame.DurabilityLevel))
writeUint16(buffer, uint16(durabilityTimeoutMillis))
}
}
if pkt.StreamIDFrame != nil {
if pkt.Magic != CmdMagicReq {
return errors.New("cannot use stream id frame in non-request packets")
}
writeFrameHeader(buffer, frameTypeReqStreamID, 2)
writeUint16(buffer, pkt.StreamIDFrame.StreamID)
}
if pkt.OpenTracingFrame != nil {
if pkt.Magic != CmdMagicReq {
return errors.New("cannot use open tracing frame in non-request packets")
}
if !c.IsFeatureEnabled(FeatureOpenTracing) {
return errors.New("cannot use open tracing frames without enabling the feature")
}
traceCtxLen := len(pkt.OpenTracingFrame.TraceContext)
writeFrameHeader(buffer, frameTypeReqOpenTracing, uint8(traceCtxLen))
buffer.Write(pkt.OpenTracingFrame.TraceContext)
}
if pkt.ServerDurationFrame != nil {
if pkt.Magic != CmdMagicRes {
return errors.New("cannot use server duration frame in non-response packets")
}
if !c.IsFeatureEnabled(FeatureDurations) {
return errors.New("cannot use server duration frames without enabling the feature")
}
writeFrameHeader(buffer, frameTypeResSrvDuration, 2)
writeUint16(buffer, EncodeSrvDura16(pkt.ServerDurationFrame.ServerDuration))
}
if pkt.UserImpersonationFrame != nil {
if pkt.Magic != CmdMagicReq {
return errors.New("cannot use user impersonation frame in non-request packets")
}
userCtxLen := len(pkt.UserImpersonationFrame.User)
writeFrameHeader(buffer, frameTypeReqUserImpersonation, uint8(userCtxLen))
buffer.Write(pkt.UserImpersonationFrame.User)
}
if pkt.PreserveExpiryFrame != nil {
if pkt.Magic != CmdMagicReq {
return errors.New("cannot use preserve expiry frame in non-request packets")
}
if !c.IsFeatureEnabled(FeaturePreserveExpiry) {
return errors.New("cannot use preserve expiry frames without enabling the feature")
}
writeFrameHeader(buffer, frameTypeReqPreserveExpiry, 0)
}
// Any frames that we don't support we'll just write to the packet, and assume that
// the user knows what they're doing re: encoding.
for _, fr := range pkt.UnsupportedFrames {
writeFrameHeader(buffer, fr.Type, uint8(len(fr.Data)))
buffer.Write(fr.Data)
}
// Copy the extras into the body of the packet
buffer.Write(extras)
// Copy the encoded key into the body of the packet
buffer.Write(encodedKey)
// Copy the value into the body of the packet
buffer.Write(pkt.Value)
n, err := c.stream.Write(buffer.Bytes())
if err != nil {
return err
}
if n != buffer.Len() {
return io.ErrShortWrite
}
return nil
}
// ReadPacket reads a packet from the network.
func (c *Conn) ReadPacket() (*Packet, int, error) {
pkt := AcquirePacket()
if c.stream == nil {
return nil, 0, io.EOF
}
// Read the entire 24-byte header first
_, err := io.ReadFull(c.stream, c.headerBuf[:])
if err != nil {
return nil, 0, err
}
// Grab the length of the full body
bodyLen := binary.BigEndian.Uint32(c.headerBuf[8:])
// Read the remaining bytes of the body
bodyBuf := make([]byte, bodyLen)
_, err = io.ReadFull(c.stream, bodyBuf)
if err != nil {
return nil, 0, err
}
pktMagic := CmdMagic(c.headerBuf[0])
switch pktMagic {
case CmdMagicReq, cmdMagicReqExt:
pkt.Magic = CmdMagicReq
pkt.Vbucket = binary.BigEndian.Uint16(c.headerBuf[6:])
case CmdMagicRes, cmdMagicResExt:
pkt.Magic = CmdMagicRes
pkt.Status = StatusCode(binary.BigEndian.Uint16(c.headerBuf[6:]))
default:
return nil, 0, errors.New("cannot decode status/vbucket for unknown packet magic")
}
pkt.Command = CmdCode(c.headerBuf[1])
pkt.Datatype = c.headerBuf[5]
pkt.Opaque = binary.BigEndian.Uint32(c.headerBuf[12:])
pkt.Cas = binary.BigEndian.Uint64(c.headerBuf[16:])
var (
extLen = int(c.headerBuf[4])
keyLen = int(binary.BigEndian.Uint16(c.headerBuf[2:]))
framesLen int
)
if pktMagic == cmdMagicReqExt || pktMagic == cmdMagicResExt {
framesLen = int(c.headerBuf[2])
keyLen = int(c.headerBuf[3])
}
if framesLen > 0 {
var (
framesBuf = bodyBuf[:framesLen]
framePos int
)
for framePos < framesLen {
frameHeader := framesBuf[framePos]
framePos++
frType := frameType((frameHeader & 0xF0) >> 4)
if frType == 15 {
frType = 15 + frameType(framesBuf[framePos])
framePos++
}
frameLen := int((frameHeader & 0x0F) >> 0)
if frameLen == 15 {
frameLen = 15 + int(framesBuf[framePos])
framePos++
}
frameBody := framesBuf[framePos : framePos+frameLen]
framePos += frameLen
switch pktMagic {
case cmdMagicReqExt:
if frType == frameTypeReqBarrier && frameLen == 0 {
pkt.BarrierFrame = &BarrierFrame{}
} else if frType == frameTypeReqSyncDurability && (frameLen == 1 || frameLen == 3) {
pkt.DurabilityLevelFrame = &DurabilityLevelFrame{
DurabilityLevel: DurabilityLevel(frameBody[0]),
}
if frameLen == 3 {
durabilityTimeoutMillis := binary.BigEndian.Uint16(frameBody[1:])
pkt.DurabilityTimeoutFrame = &DurabilityTimeoutFrame{
DurabilityTimeout: time.Duration(durabilityTimeoutMillis) * time.Millisecond,
}
} else {
// We follow the semantic that duplicate frames overwrite previous ones,
// since the timeout frame is 'virtual' to us, we need to clear it in case
// this is a duplicate frame.
pkt.DurabilityTimeoutFrame = nil
}
} else if frType == frameTypeReqStreamID && frameLen == 2 {
pkt.StreamIDFrame = &StreamIDFrame{
StreamID: binary.BigEndian.Uint16(frameBody),
}
} else if frType == frameTypeReqOpenTracing {
pkt.OpenTracingFrame = &OpenTracingFrame{
TraceContext: frameBody,
}
} else if frType == frameTypeReqPreserveExpiry {
pkt.PreserveExpiryFrame = &PreserveExpiryFrame{}
} else if frType == frameTypeReqUserImpersonation {
pkt.UserImpersonationFrame = &UserImpersonationFrame{
User: frameBody,
}
} else {
// If we don't understand this frame type, we record it as an
// UnsupportedFrame (as opposed to dropping it blindly)
pkt.UnsupportedFrames = append(pkt.UnsupportedFrames, UnsupportedFrame{
Type: frType,
Data: frameBody,
})
}
case cmdMagicResExt:
if frType == frameTypeResSrvDuration && frameLen == 2 {
serverDurationEnc := binary.BigEndian.Uint16(frameBody)
pkt.ServerDurationFrame = &ServerDurationFrame{
ServerDuration: DecodeSrvDura16(serverDurationEnc),
}
} else if frType == frameTypeResReadUnits && frameLen == 2 {
pkt.ReadUnitsFrame = &ReadUnitsFrame{
ReadUnits: binary.BigEndian.Uint16(frameBody),
}
} else if frType == frameTypeResWriteUnits && frameLen == 2 {
pkt.WriteUnitsFrame = &WriteUnitsFrame{
WriteUnits: binary.BigEndian.Uint16(frameBody),
}
} else {
// If we don't understand this frame type, we record it as an
// UnsupportedFrame (as opposed to dropping it blindly)
pkt.UnsupportedFrames = append(pkt.UnsupportedFrames, UnsupportedFrame{
Type: frType,
Data: frameBody,
})
}
default:
return nil, 0, errors.New("got unexpected magic when decoding frames")
}
}
}
pkt.Extras = bodyBuf[framesLen : framesLen+extLen]
pkt.Key = bodyBuf[framesLen+extLen : framesLen+extLen+keyLen]
pkt.Value = bodyBuf[framesLen+extLen+keyLen:]
if c.isCollectionsEnabled() {
if pkt.Command == CmdObserve {
// While it's possible that the Observe operation is in fact supported with collections
// enabled, we don't currently implement that operation for simplicity, as the key is
// actually hidden away in the value data instead of the usual key data.
return nil, 0, errors.New("the observe operation is not supported with collections enabled")
}
if keyLen > 0 && IsCommandCollectionEncoded(pkt.Command) {
collectionID, idLen, err := DecodeULEB128_32(pkt.Key)
if err != nil {
return nil, 0, err
}
pkt.Key = pkt.Key[idLen:]
pkt.CollectionID = collectionID
}
}
return pkt, 24 + int(bodyLen), nil
}
// writeUint16 - Similar to 'bytes.BigEndian.PutUint16' accept we write directly into the provided buffer.
func writeUint16(buffer *bytes.Buffer, n uint16) {
buffer.WriteByte(byte(n >> 8))
buffer.WriteByte(byte(n))
}
// writeUint32 - Similar to 'bytes.BigEndian.PutUint32' accept we write directly into the provided buffer.
func writeUint32(buffer *bytes.Buffer, n uint32) {
buffer.WriteByte(byte(n >> 24))
buffer.WriteByte(byte(n >> 16))
buffer.WriteByte(byte(n >> 8))
buffer.WriteByte(byte(n))
}
// writeUint64 - Similar to 'bytes.BigEndian.PutUint64' accept we write directly into the provided buffer.
func writeUint64(buffer *bytes.Buffer, n uint64) {
buffer.WriteByte(byte(n >> 56))
buffer.WriteByte(byte(n >> 48))
buffer.WriteByte(byte(n >> 40))
buffer.WriteByte(byte(n >> 32))
buffer.WriteByte(byte(n >> 24))
buffer.WriteByte(byte(n >> 16))
buffer.WriteByte(byte(n >> 8))
buffer.WriteByte(byte(n))
}
// writeFrameHeader - Write a single byte containing information about the following frame directly into the provided
// buffer.
func writeFrameHeader(buffer *bytes.Buffer, frameType frameType, frameLen uint8) {
if frameLen < 15 {
buffer.WriteByte(uint8(frameType)<<4 | frameLen)
return
}
buffer.WriteByte(uint8(frameType)<<4 | 15)
buffer.WriteByte(frameLen - 15)
}
// calcHeaderSize calculates the correct length header for a frame of variable size.
func calcHeaderSize(frameLen int) int {
if frameLen < 15 {
return 1 + frameLen
}
return 2 + frameLen
}
gocbcore-10.2.3/memd/conn_test.go 0000664 0000000 0000000 00000012637 14417540156 0016645 0 ustar 00root root 0000000 0000000 package memd
import (
"bytes"
"reflect"
"testing"
"time"
)
func testPktRoundTrip(t *testing.T, pkt *Packet, features []HelloFeature) {
t.Helper()
// Create a buffer and connection for testing
buf := &bytes.Buffer{}
conn := NewConn(buf)
// Enable the specific features
for _, feature := range features {
conn.EnableFeature(feature)
}
// Write our packet to the connection
err := conn.WritePacket(pkt)
if err != nil {
t.Fatalf("packet writing failed: %s", err)
}
// Read the packet back
pktOut, _, err := conn.ReadPacket()
if err != nil {
t.Fatalf("packet reading failed: %s", err)
}
// Check that the packet matched like we expect
if !reflect.DeepEqual(pkt, pktOut) {
t.Errorf("packets did not match after roundtrip\n"+
"EXP: %+v\nGOT: %+v",
pkt, pktOut)
t.Logf("EXP DURLVL: %+v\nGOT DURLVL: %+v", pkt.DurabilityLevelFrame, pktOut.DurabilityLevelFrame)
t.Logf("EXP DURATM: %+v\nGOT DURATM: %+v", pkt.DurabilityTimeoutFrame, pktOut.DurabilityTimeoutFrame)
t.Logf("EXP STRMID: %+v\nGOT STRMID: %+v", pkt.StreamIDFrame, pktOut.StreamIDFrame)
t.Logf("EXP OTRCTX: %+v\nGOT OTRCTX: %+v", pkt.OpenTracingFrame, pktOut.OpenTracingFrame)
t.Logf("EXP SRVDUR: %+v\nGOT SRVDUR: %+v", pkt.ServerDurationFrame, pktOut.ServerDurationFrame)
t.Logf("EXP USERIMP: %+v\nGOT USERIMP: %+v", pkt.UserImpersonationFrame, pktOut.UserImpersonationFrame)
t.Logf("EXP UNSPPTD: %+v\nGOT UNSPPTD: %+v", pkt.UnsupportedFrames, pktOut.UnsupportedFrames)
t.FailNow()
}
}
var noFeatures = []HelloFeature{}
var allFeatures = []HelloFeature{
FeatureDatatype,
FeatureTLS,
FeatureTCPNoDelay,
FeatureSeqNo,
FeatureTCPDelay,
FeatureXattr,
FeatureXerror,
FeatureSelectBucket,
FeatureSnappy,
FeatureJSON,
FeatureDuplex,
FeatureClusterMapNotif,
FeatureUnorderedExec,
FeatureDurations,
FeatureAltRequests,
FeatureSyncReplication,
FeatureCollections,
FeatureOpenTracing,
FeaturePreserveExpiry,
}
func TestPktRtBasicReq(t *testing.T) {
testPktRoundTrip(t, &Packet{
Magic: CmdMagicReq,
Command: CmdGetErrorMap,
Datatype: 0x22,
Vbucket: 0x9f9e,
Opaque: 0x87654321,
Cas: 0x7654321076543210,
Key: []byte("Hello"),
Extras: []byte("I am some data which is longer?"),
Value: []byte("World"),
}, noFeatures)
}
func TestPktRtBasicRes(t *testing.T) {
testPktRoundTrip(t, &Packet{
Magic: CmdMagicRes,
Command: CmdGetErrorMap,
Datatype: 0x22,
Status: StatusBusy,
Opaque: 0x87654321,
Cas: 0x7654321076543210,
Key: []byte("Hello"),
Extras: []byte("I am some data which is longer?"),
Value: []byte("World"),
}, noFeatures)
}
func TestPktRtBasicReqExt(t *testing.T) {
testPktRoundTrip(t, &Packet{
Magic: CmdMagicReq,
Command: CmdGAT,
Datatype: 0x22,
Vbucket: 0x9f9e,
Opaque: 0x87654321,
Cas: 0x7654321076543210,
CollectionID: 99,
Key: []byte("Hello"),
Extras: []byte("I am some data which is longer?"),
Value: []byte("World"),
BarrierFrame: &BarrierFrame{},
DurabilityLevelFrame: &DurabilityLevelFrame{
DurabilityLevel: DurabilityLevelPersistToMajority,
},
DurabilityTimeoutFrame: &DurabilityTimeoutFrame{
DurabilityTimeout: 2 * time.Second,
},
StreamIDFrame: &StreamIDFrame{
StreamID: 0xe1f8,
},
OpenTracingFrame: &OpenTracingFrame{
TraceContext: []byte("This is some data longer than 15bytes"),
},
UserImpersonationFrame: &UserImpersonationFrame{
User: []byte("barry"),
},
PreserveExpiryFrame: &PreserveExpiryFrame{},
}, allFeatures)
}
func TestPktRtBasicResExt(t *testing.T) {
testPktRoundTrip(t, &Packet{
Magic: CmdMagicRes,
Command: CmdGAT,
Datatype: 0x22,
Status: StatusBusy,
Opaque: 0x87654321,
Cas: 0x7654321076543210,
CollectionID: 99,
Key: []byte("Hello"),
Extras: []byte("I am some data which is longer?"),
Value: []byte("World"),
ServerDurationFrame: &ServerDurationFrame{
ServerDuration: 119973 * time.Microsecond,
},
}, allFeatures)
}
func TestPktUnsupportedFrameReqExt(t *testing.T) {
testPktRoundTrip(t, &Packet{
Magic: CmdMagicReq,
Command: CmdGAT,
Datatype: 0x22,
Vbucket: 0x9f9e,
Opaque: 0x87654321,
Cas: 0x7654321076543210,
Key: []byte("Hello"),
Extras: []byte("I am some data which is longer?"),
Value: []byte("World"),
UnsupportedFrames: []UnsupportedFrame{
{
Type: 13,
Data: []byte("barry"),
},
},
}, allFeatures)
}
func TestPktUnsupportedFramesReqExt(t *testing.T) {
testPktRoundTrip(t, &Packet{
Magic: CmdMagicReq,
Command: CmdGAT,
Datatype: 0x22,
Vbucket: 0x9f9e,
Opaque: 0x87654321,
Cas: 0x7654321076543210,
Key: []byte("Hello"),
Extras: []byte("I am some data which is longer?"),
Value: []byte("World"),
UnsupportedFrames: []UnsupportedFrame{
{
Type: 13,
Data: []byte("barry"),
},
{
Type: 12,
Data: []byte("barrysmate"),
},
{
Type: 11,
Data: []byte("barrysothermatewithareallylongname"),
},
},
}, allFeatures)
}
func TestPktUnsupportedFrameResExt(t *testing.T) {
testPktRoundTrip(t, &Packet{
Magic: CmdMagicRes,
Command: CmdGAT,
Datatype: 0x22,
Opaque: 0x87654321,
Cas: 0x7654321076543210,
Key: []byte("Hello"),
Extras: []byte("I am some data which is longer?"),
Value: []byte("World"),
UnsupportedFrames: []UnsupportedFrame{
{
Type: 13,
Data: []byte("barry"),
},
},
}, allFeatures)
}
gocbcore-10.2.3/memd/constants.go 0000664 0000000 0000000 00000036026 14417540156 0016663 0 ustar 00root root 0000000 0000000 package memd
import "fmt"
// CmdMagic represents the magic number that begins the header
// of every packet and informs the rest of the header format.
type CmdMagic uint8
const (
// CmdMagicReq indicates that the packet is a request.
CmdMagicReq = CmdMagic(0x80)
// CmdMagicRes indicates that the packet is a response.
CmdMagicRes = CmdMagic(0x81)
// These are private rather than public as the library will automatically
// switch to and from these magics based on the use of frames within a packet.
cmdMagicReqExt = CmdMagic(0x08)
cmdMagicResExt = CmdMagic(0x18)
)
// frameType specifies which kind of frame extra a particular block belongs to.
// This is a private type since we automatically encode this internally based on
// whether the specific frame block is attached to the packet.
type frameType uint8
const (
frameTypeReqBarrier = frameType(0)
frameTypeReqSyncDurability = frameType(1)
frameTypeReqStreamID = frameType(2)
frameTypeReqOpenTracing = frameType(3)
frameTypeReqUserImpersonation = frameType(4)
frameTypeReqPreserveExpiry = frameType(5)
frameTypeResSrvDuration = frameType(0)
frameTypeResReadUnits = frameType(1)
frameTypeResWriteUnits = frameType(2)
)
// HelloFeature represents a feature code included in a memcached
// HELLO operation.
type HelloFeature uint16
const (
// FeatureDatatype indicates support for Datatype fields.
FeatureDatatype = HelloFeature(0x01)
// FeatureTLS indicates support for TLS
FeatureTLS = HelloFeature(0x02)
// FeatureTCPNoDelay indicates support for TCP no-delay.
FeatureTCPNoDelay = HelloFeature(0x03)
// FeatureSeqNo indicates support for mutation tokens.
FeatureSeqNo = HelloFeature(0x04)
// FeatureTCPDelay indicates support for TCP delay.
FeatureTCPDelay = HelloFeature(0x05)
// FeatureXattr indicates support for document xattrs.
FeatureXattr = HelloFeature(0x06)
// FeatureXerror indicates support for extended errors.
FeatureXerror = HelloFeature(0x07)
// FeatureSelectBucket indicates support for the SelectBucket operation.
FeatureSelectBucket = HelloFeature(0x08)
// Feature 0x09 is reserved and cannot be used.
// FeatureSnappy indicates support for snappy compressed documents.
FeatureSnappy = HelloFeature(0x0a)
// FeatureJSON indicates support for JSON datatype data.
FeatureJSON = HelloFeature(0x0b)
// FeatureDuplex indicates support for duplex communications.
FeatureDuplex = HelloFeature(0x0c)
// FeatureClusterMapNotif indicates support for cluster-map update notifications.
FeatureClusterMapNotif = HelloFeature(0x0d)
// FeatureUnorderedExec indicates support for unordered execution of operations.
FeatureUnorderedExec = HelloFeature(0x0e)
// FeatureDurations indicates support for server durations.
FeatureDurations = HelloFeature(0xf)
// FeatureAltRequests indicates support for requests with flexible frame extras.
FeatureAltRequests = HelloFeature(0x10)
// FeatureSyncReplication indicates support for requests synchronous durability requirements.
FeatureSyncReplication = HelloFeature(0x11)
// FeatureCollections indicates support for collections.
FeatureCollections = HelloFeature(0x12)
// FeatureOpenTracing indicates support for OpenTracing.
FeatureOpenTracing = HelloFeature(0x13)
// FeaturePreserveExpiry indicates support for preserve TTL.
FeaturePreserveExpiry = HelloFeature(0x14)
// FeaturePITR indicates support for PITR snapshots.
FeaturePITR = HelloFeature(0x16)
// FeatureCreateAsDeleted indicates support for the create as deleted feature.
FeatureCreateAsDeleted = HelloFeature(0x17)
// FeatureReplaceBodyWithXattr indicates support for the replace body with xattr feature.
FeatureReplaceBodyWithXattr = HelloFeature(0x19)
FeatureResourceUnits = HelloFeature(0x1a)
)
// StreamEndStatus represents the reason for a DCP stream ending
type StreamEndStatus uint32
const (
// StreamEndOK represents that the stream ended successfully.
StreamEndOK = StreamEndStatus(0x00)
// StreamEndClosed represents that the stream was forcefully closed.
StreamEndClosed = StreamEndStatus(0x01)
// StreamEndStateChanged represents that the stream was closed due to a state change.
StreamEndStateChanged = StreamEndStatus(0x02)
// StreamEndDisconnected represents that the stream was closed due to disconnection.
StreamEndDisconnected = StreamEndStatus(0x03)
// StreamEndTooSlow represents that the stream was closed due to the stream being too slow.
StreamEndTooSlow = StreamEndStatus(0x04)
// StreamEndBackfillFailed represents that the stream was closed due to backfill failing.
StreamEndBackfillFailed = StreamEndStatus(0x05)
// StreamEndFilterEmpty represents that the stream was closed due to the filter being empty.
StreamEndFilterEmpty = StreamEndStatus(0x07)
)
// KVText returns the textual representation of this StreamEndStatus.
func (code StreamEndStatus) KVText() string {
switch code {
case StreamEndOK:
return "success"
case StreamEndClosed:
return "stream closed"
case StreamEndStateChanged:
return "state changed"
case StreamEndDisconnected:
return "disconnected"
case StreamEndTooSlow:
return "too slow"
case StreamEndFilterEmpty:
return "filter empty"
case StreamEndBackfillFailed:
return "backfill failed"
default:
return fmt.Sprintf("unknown stream close reason (%d)", code)
}
}
// StreamEventCode is the code for a DCP Stream event
type StreamEventCode uint32
const (
// StreamEventCollectionCreate is the StreamEventCode for a collection create event
StreamEventCollectionCreate = StreamEventCode(0x00)
// StreamEventCollectionDelete is the StreamEventCode for a collection delete event
StreamEventCollectionDelete = StreamEventCode(0x01)
// StreamEventCollectionFlush is the StreamEventCode for a collection flush event
StreamEventCollectionFlush = StreamEventCode(0x02)
// StreamEventScopeCreate is the StreamEventCode for a scope create event
StreamEventScopeCreate = StreamEventCode(0x03)
// StreamEventScopeDelete is the StreamEventCode for a scope delete event
StreamEventScopeDelete = StreamEventCode(0x04)
// StreamEventCollectionChanged is the StreamEventCode for a collection changed event
StreamEventCollectionChanged = StreamEventCode(0x05)
)
// VbucketState represents the state of a particular vbucket on a particular server.
type VbucketState uint32
const (
// VbucketStateActive indicates the vbucket is active on this server
VbucketStateActive = VbucketState(0x01)
// VbucketStateReplica indicates the vbucket is a replica on this server
VbucketStateReplica = VbucketState(0x02)
// VbucketStatePending indicates the vbucket is preparing to become active on this server.
VbucketStatePending = VbucketState(0x03)
// VbucketStateDead indicates the vbucket is no longer valid on this server.
VbucketStateDead = VbucketState(0x04)
)
// SetMetaOption represents possible option values for a SetMeta operation.
type SetMetaOption uint32
const (
// ForceMetaOp disables conflict resolution for the document and allows the
// operation to be applied to an active, pending, or replica vbucket.
ForceMetaOp = SetMetaOption(0x01)
// UseLwwConflictResolution switches to Last-Write-Wins conflict resolution
// for the document.
UseLwwConflictResolution = SetMetaOption(0x02)
// RegenerateCas causes the server to invalidate the current CAS value for
// a document, and to generate a new one.
RegenerateCas = SetMetaOption(0x04)
// SkipConflictResolution disables conflict resolution for the document.
SkipConflictResolution = SetMetaOption(0x08)
// IsExpiration indicates that the message is for an expired document.
IsExpiration = SetMetaOption(0x10)
)
// KeyState represents the various storage states of a key on the server.
type KeyState uint8
const (
// KeyStateNotPersisted indicates the key is in memory, but not yet written to disk.
KeyStateNotPersisted = KeyState(0x00)
// KeyStatePersisted indicates that the key has been written to disk.
KeyStatePersisted = KeyState(0x01)
// KeyStateNotFound indicates that the key is not found in memory or on disk.
KeyStateNotFound = KeyState(0x80)
// KeyStateDeleted indicates that the key has been written to disk as deleted.
KeyStateDeleted = KeyState(0x81)
)
// SubDocOpType specifies the type of a sub-document operation.
type SubDocOpType uint8
const (
// SubDocOpGet indicates the operation is a sub-document `Get` operation.
SubDocOpGet = SubDocOpType(CmdSubDocGet)
// SubDocOpExists indicates the operation is a sub-document `Exists` operation.
SubDocOpExists = SubDocOpType(CmdSubDocExists)
// SubDocOpGetCount indicates the operation is a sub-document `GetCount` operation.
SubDocOpGetCount = SubDocOpType(CmdSubDocGetCount)
// SubDocOpDictAdd indicates the operation is a sub-document `Add` operation.
SubDocOpDictAdd = SubDocOpType(CmdSubDocDictAdd)
// SubDocOpDictSet indicates the operation is a sub-document `Set` operation.
SubDocOpDictSet = SubDocOpType(CmdSubDocDictSet)
// SubDocOpDelete indicates the operation is a sub-document `Remove` operation.
SubDocOpDelete = SubDocOpType(CmdSubDocDelete)
// SubDocOpReplace indicates the operation is a sub-document `Replace` operation.
SubDocOpReplace = SubDocOpType(CmdSubDocReplace)
// SubDocOpArrayPushLast indicates the operation is a sub-document `ArrayPushLast` operation.
SubDocOpArrayPushLast = SubDocOpType(CmdSubDocArrayPushLast)
// SubDocOpArrayPushFirst indicates the operation is a sub-document `ArrayPushFirst` operation.
SubDocOpArrayPushFirst = SubDocOpType(CmdSubDocArrayPushFirst)
// SubDocOpArrayInsert indicates the operation is a sub-document `ArrayInsert` operation.
SubDocOpArrayInsert = SubDocOpType(CmdSubDocArrayInsert)
// SubDocOpArrayAddUnique indicates the operation is a sub-document `ArrayAddUnique` operation.
SubDocOpArrayAddUnique = SubDocOpType(CmdSubDocArrayAddUnique)
// SubDocOpCounter indicates the operation is a sub-document `Counter` operation.
SubDocOpCounter = SubDocOpType(CmdSubDocCounter)
// SubDocOpGetDoc represents a full document retrieval, for use with extended attribute ops.
SubDocOpGetDoc = SubDocOpType(CmdGet)
// SubDocOpSetDoc represents a full document set, for use with extended attribute ops.
SubDocOpSetDoc = SubDocOpType(CmdSet)
// SubDocOpAddDoc represents a full document add, for use with extended attribute ops.
SubDocOpAddDoc = SubDocOpType(CmdAdd)
// SubDocOpDeleteDoc represents a full document delete, for use with extended attribute ops.
SubDocOpDeleteDoc = SubDocOpType(CmdDelete)
// SubDocOpReplaceBodyWithXattr represents a replace body with xattr op.
// Uncommitted: This API may change in the future.
SubDocOpReplaceBodyWithXattr = SubDocOpType(CmdSubDocReplaceBodyWithXattr)
)
// DcpOpenFlag specifies flags for DCP connections configured when the stream is opened.
type DcpOpenFlag uint32
const (
// DcpOpenFlagProducer indicates this connection wants the other end to be a producer.
DcpOpenFlagProducer = DcpOpenFlag(0x01)
// DcpOpenFlagNotifier indicates this connection wants the other end to be a notifier.
DcpOpenFlagNotifier = DcpOpenFlag(0x02)
// DcpOpenFlagIncludeXattrs indicates the client wishes to receive extended attributes.
DcpOpenFlagIncludeXattrs = DcpOpenFlag(0x04)
// DcpOpenFlagNoValue indicates the client does not wish to receive mutation values.
DcpOpenFlagNoValue = DcpOpenFlag(0x08)
// DcpOpenFlagIncludeDeleteTimes indicates the client wishes to receive delete times.
DcpOpenFlagIncludeDeleteTimes = DcpOpenFlag(0x20)
// DcpOpenFlagPiTR indicates the client wishes to receive PITR snapshots
DcpOpenFlagPiTR = DcpOpenFlag(0x80)
)
// DcpStreamAddFlag specifies flags for DCP streams configured when the stream is opened.
type DcpStreamAddFlag uint32
const (
// DcpStreamAddFlagDiskOnly indicates that stream should only send items if they are on disk
DcpStreamAddFlagDiskOnly = DcpStreamAddFlag(0x02)
// DcpStreamAddFlagLatest indicates this stream wants to get data up to the latest seqno.
DcpStreamAddFlagLatest = DcpStreamAddFlag(0x04)
// DcpStreamAddFlagActiveOnly indicates this stream should only connect to an active vbucket.
DcpStreamAddFlagActiveOnly = DcpStreamAddFlag(0x10)
// DcpStreamAddFlagStrictVBUUID indicates the vbuuid must match unless the start seqno
// is 0 and the vbuuid is also 0.
DcpStreamAddFlagStrictVBUUID = DcpStreamAddFlag(0x20)
)
// DatatypeFlag specifies data flags for the value of a document.
type DatatypeFlag uint8
const (
// DatatypeFlagJSON indicates the server believes the value payload to be JSON.
DatatypeFlagJSON = DatatypeFlag(0x01)
// DatatypeFlagCompressed indicates the value payload is compressed.
DatatypeFlagCompressed = DatatypeFlag(0x02)
// DatatypeFlagXattrs indicates the inclusion of xattr data in the value payload.
DatatypeFlagXattrs = DatatypeFlag(0x04)
)
// SubdocFlag specifies flags for a sub-document operation.
type SubdocFlag uint8
const (
// SubdocFlagNone indicates no special treatment for this operation.
SubdocFlagNone = SubdocFlag(0x00)
// SubdocFlagMkDirP indicates that the path should be created if it does not already exist.
SubdocFlagMkDirP = SubdocFlag(0x01)
// 0x02 is unused, formally SubdocFlagMkDoc
// SubdocFlagXattrPath indicates that the path refers to an Xattr rather than the document body.
SubdocFlagXattrPath = SubdocFlag(0x04)
// 0x08 is unused, formally SubdocFlagAccessDeleted
// SubdocFlagExpandMacros indicates that the value portion of any sub-document mutations
// should be expanded if they contain macros such as ${Mutation.CAS}.
SubdocFlagExpandMacros = SubdocFlag(0x10)
)
// SubdocDocFlag specifies document-level flags for a sub-document operation.
type SubdocDocFlag uint8
const (
// SubdocDocFlagNone indicates no special treatment for this operation.
SubdocDocFlagNone = SubdocDocFlag(0x00)
// SubdocDocFlagMkDoc indicates that the document should be created if it does not already exist.
SubdocDocFlagMkDoc = SubdocDocFlag(0x01)
// SubdocDocFlagAddDoc indices that this operation should be an add rather than set.
SubdocDocFlagAddDoc = SubdocDocFlag(0x02)
// SubdocDocFlagAccessDeleted indicates that you wish to receive soft-deleted documents.
// Internal: This should never be used and is not supported.
SubdocDocFlagAccessDeleted = SubdocDocFlag(0x04)
// SubdocDocFlagCreateAsDeleted indicates that the document should be created as deleted.
// That is, to create a tombstone only.
// Internal: This should never be used and is not supported.
SubdocDocFlagCreateAsDeleted = SubdocDocFlag(0x08)
)
// DurabilityLevel specifies the level to use for enhanced durability requirements.
type DurabilityLevel uint8
const (
// DurabilityLevelMajority specifies that a change must be replicated to (held in memory)
// a majority of the nodes for the bucket.
DurabilityLevelMajority = DurabilityLevel(0x01)
// DurabilityLevelMajorityAndPersistOnMaster specifies that a change must be replicated to (held in memory)
// a majority of the nodes for the bucket and additionally persisted to disk on the active node.
DurabilityLevelMajorityAndPersistOnMaster = DurabilityLevel(0x02)
// DurabilityLevelPersistToMajority specifies that a change must be persisted to (written to disk)
// a majority for the bucket.
DurabilityLevelPersistToMajority = DurabilityLevel(0x03)
)
gocbcore-10.2.3/memd/packet.go 0000664 0000000 0000000 00000014530 14417540156 0016112 0 ustar 00root root 0000000 0000000 package memd
import (
"bytes"
"fmt"
"sync"
"time"
)
// BarrierFrame is used to signal to the server that this command should be
// barriered and must not be executed concurrently with other commands.
type BarrierFrame struct {
// Barrier frames have no additional configuration, but their existence
// triggers the barriering behaviour.
}
// DurabilityLevelFrame allows you to specify a durability level for an
// operation through the frame extras.
type DurabilityLevelFrame struct {
DurabilityLevel DurabilityLevel
}
// DurabilityTimeoutFrame allows you to specify a specific timeout for
// durability operations to timeout. Note that this frame is actually
// an extension of DurabilityLevelFrame and requires that frame to also
// be used in order to function.
type DurabilityTimeoutFrame struct {
DurabilityTimeout time.Duration
}
// StreamIDFrame provides information about which stream this particular
// operation is related to (used for DCP streams).
type StreamIDFrame struct {
StreamID uint16
}
// OpenTracingFrame allows open tracing context information to be included
// along with a command which is being performed.
type OpenTracingFrame struct {
TraceContext []byte
}
// ServerDurationFrame allows the server to return information about the
// period of time an operation took to complete.
type ServerDurationFrame struct {
ServerDuration time.Duration
}
// UnsupportedFrame is used to include an unsupported frame type in the
// packet data to enable further processing if needed.
type UnsupportedFrame struct {
Type frameType
Data []byte
}
// UserImpersonationFrame is used to indicate a user to impersonate.
// Internal: This should never be used and is not supported.
type UserImpersonationFrame struct {
User []byte
}
// PreserveExpiryFrame is used to indicate that the server should preserve the
// expiry time for existing document.
type PreserveExpiryFrame struct {
// Preserve Expiry frames have no extra configuration, but their existence
// triggers the preserve expiry behaviour.
}
// ReadUnitsFrame allows the server to return information about the
// number of read units used by a command.
type ReadUnitsFrame struct {
ReadUnits uint16
}
// WriteUnitsFrame allows the server to return information about the
// number of write units used by a command.
type WriteUnitsFrame struct {
WriteUnits uint16
}
// Packet represents a single request or response packet being exchanged
// between two clients.
type Packet struct {
Magic CmdMagic
Command CmdCode
Datatype uint8
Status StatusCode
Vbucket uint16
Opaque uint32
Cas uint64
CollectionID uint32
Key []byte
Extras []byte
Value []byte
BarrierFrame *BarrierFrame
DurabilityLevelFrame *DurabilityLevelFrame
DurabilityTimeoutFrame *DurabilityTimeoutFrame
StreamIDFrame *StreamIDFrame
OpenTracingFrame *OpenTracingFrame
ServerDurationFrame *ServerDurationFrame
UserImpersonationFrame *UserImpersonationFrame
PreserveExpiryFrame *PreserveExpiryFrame
ReadUnitsFrame *ReadUnitsFrame
WriteUnitsFrame *WriteUnitsFrame
UnsupportedFrames []UnsupportedFrame
}
func (pak *Packet) String() string {
var buffer bytes.Buffer
fmt.Fprintf(
&buffer,
"memd.Packet{Magic:%#02x(%s), Command:%#02x(%s), Datatype:%#02x, Status:%#04x(%s), Vbucket:%d(%#04x), Opaque:%#08x, "+
"Cas: %#08x, CollectionID:%d(%#08x), Barrier:%t\nKey:\n%sValue:\n%sExtras:\n%s",
uint8(pak.Magic),
pak.Magic,
pak.Command,
pak.Command.Name(),
pak.Datatype,
uint16(pak.Status),
pak.Status,
pak.Vbucket,
pak.Vbucket,
pak.Opaque,
pak.Cas,
pak.CollectionID,
pak.CollectionID,
pak.BarrierFrame != nil,
bytesToHexAsciiString(pak.Key),
bytesToHexAsciiString(pak.Value),
bytesToHexAsciiString(pak.Extras),
)
if pak.DurabilityLevelFrame != nil {
fmt.Fprintf(&buffer, "\nDurability Level: %#02x", pak.DurabilityLevelFrame.DurabilityLevel)
if pak.DurabilityTimeoutFrame != nil {
fmt.Fprintf(&buffer, "\nDurability Level Timeout: %s", pak.DurabilityTimeoutFrame.DurabilityTimeout)
}
}
if pak.StreamIDFrame != nil {
fmt.Fprintf(&buffer, "\nStreamID: %#02x", pak.StreamIDFrame.StreamID)
}
if pak.OpenTracingFrame != nil {
fmt.Fprintf(&buffer, "\nTrace Context:\n%s", bytesToHexAsciiString(pak.OpenTracingFrame.TraceContext))
}
if pak.ServerDurationFrame != nil {
fmt.Fprintf(&buffer, "\nServer Duration: %s", pak.ServerDurationFrame.ServerDuration)
}
if pak.UserImpersonationFrame != nil {
fmt.Fprintf(&buffer, "\nUser: %s", string(pak.UserImpersonationFrame.User))
}
if pak.PreserveExpiryFrame != nil {
fmt.Fprintf(&buffer, "\nPreserve Expiry: true")
}
if len(pak.UnsupportedFrames) > 0 {
fmt.Fprintf(&buffer, "\nUnsupported frames:")
for _, frame := range pak.UnsupportedFrames {
fmt.Fprintf(&buffer, "\nFrame type: %02x, data: %s", frame.Type, bytesToHexAsciiString(frame.Data))
}
}
fmt.Fprintf(&buffer, "}")
return buffer.String()
}
func bytesToHexAsciiString(bytes []byte) string {
out := ""
var ascii [16]byte
n := (len(bytes) + 15) &^ 15
for i := 0; i < n; i++ {
// include the line numbering at beginning of every line
if i%16 == 0 {
out += fmt.Sprintf("%4d", i)
}
// extra space between blocks of 8 bytes
if i%8 == 0 {
out += " "
}
// if we have bytes left, print the hex
if i < len(bytes) {
out += fmt.Sprintf(" %02X", bytes[i])
} else {
out += " "
}
// build the ascii
if i >= len(bytes) {
ascii[i%16] = ' '
} else if bytes[i] < 32 || bytes[i] > 126 {
ascii[i%16] = '.'
} else {
ascii[i%16] = bytes[i]
}
// at the end of the line, print the newline.
if i%16 == 15 {
out += fmt.Sprintf(" %s\n", string(ascii[:]))
}
}
return out
}
// packetPool - Thread safe pool containing memcached packet structures. Used by the memcached connection when reading
// packets from the TCP socket.
var packetPool = sync.Pool{
New: func() interface{} {
return &Packet{}
},
}
// AcquirePacket - Retrieve a packet from the internal pool. Note that the packet should be returned to the pool to
// avoid unnecessary allocations.
func AcquirePacket() *Packet {
return packetPool.Get().(*Packet)
}
// ReleasePacket - Return a packet to the internal pool. Note that the packet will be reset, removing any active
// pointers to existing data structures.
func ReleasePacket(packet *Packet) {
*packet = Packet{}
packetPool.Put(packet)
}
gocbcore-10.2.3/memd/packet_test.go 0000664 0000000 0000000 00000006176 14417540156 0017160 0 ustar 00root root 0000000 0000000 package memd
import (
"strings"
"testing"
"time"
)
func TestPacketString(t *testing.T) {
pak := &Packet{
Magic: CmdMagicReq,
Command: CmdGetErrorMap,
Datatype: 0x22,
Vbucket: 0x9f9e,
Opaque: 0x87654321,
Cas: 0x7654321076543210,
CollectionID: 0x08,
Key: []byte("Hello"),
Extras: []byte("I am some data which is longer?"),
Value: []byte("World"),
}
toStr := pak.String()
expected := "memd.Packet{Magic:0x80(CmdMagicReq), Command:0xfe(CMD_GETERRORMAP), Datatype:0x22, Status:0x0000(success), " +
"Vbucket:40862(0x9f9e), Opaque:0x87654321, Cas: 0x7654321076543210, CollectionID:8(0x00000008), Barrier:false" +
"\nKey:\n" +
" 0 48 65 6C 6C 6F Hello " +
"\nValue:\n" +
" 0 57 6F 72 6C 64 World " +
"\nExtras:\n" +
" 0 49 20 61 6D 20 73 6F 6D 65 20 64 61 74 61 20 77 I am some data w\n" +
" 16 68 69 63 68 20 69 73 20 6C 6F 6E 67 65 72 3F hich is longer? \n" +
"}"
if !strings.EqualFold(expected, toStr) {
t.Fatalf("Expected packet string value of \n%v\n did not match actual \n%v", expected, toStr)
}
}
func TestPacketStringFramingExtras(t *testing.T) {
pak := &Packet{
Magic: CmdMagicReq,
Command: CmdGetErrorMap,
Datatype: 0x22,
Vbucket: 0x9f9e,
Opaque: 0x87654321,
Cas: 0x7654321076543210,
CollectionID: 0x08,
Key: []byte("Hello"),
Value: []byte("World"),
DurabilityLevelFrame: &DurabilityLevelFrame{
DurabilityLevel: DurabilityLevelPersistToMajority,
},
DurabilityTimeoutFrame: &DurabilityTimeoutFrame{
DurabilityTimeout: 10 * time.Second,
},
StreamIDFrame: &StreamIDFrame{
StreamID: 0x05,
},
OpenTracingFrame: &OpenTracingFrame{
TraceContext: []byte("This is some data longer than 15bytes"),
},
ServerDurationFrame: &ServerDurationFrame{
ServerDuration: 1 * time.Millisecond,
},
UserImpersonationFrame: &UserImpersonationFrame{
User: []byte("system"),
},
PreserveExpiryFrame: &PreserveExpiryFrame{},
}
toStr := pak.String()
expected := "memd.Packet{Magic:0x80(CmdMagicReq), Command:0xfe(CMD_GETERRORMAP), Datatype:0x22, Status:0x0000(success), " +
"Vbucket:40862(0x9f9e), Opaque:0x87654321, Cas: 0x7654321076543210, CollectionID:8(0x00000008), Barrier:false" +
"\nKey:\n" +
" 0 48 65 6C 6C 6F Hello " +
"\nValue:\n" +
" 0 57 6F 72 6C 64 World " +
"\nExtras:\n" +
"\nDurability Level: 0x03" +
"\nDurability Level Timeout: 10s" +
"\nStreamID: 0x05" +
"\nTrace Context:\n" +
" 0 54 68 69 73 20 69 73 20 73 6F 6D 65 20 64 61 74 This is some dat\n" +
" 16 61 20 6C 6F 6E 67 65 72 20 74 68 61 6E 20 31 35 a longer than 15\n" +
" 32 62 79 74 65 73 bytes \n" +
"\nServer Duration: 1ms" +
"\nUser: system" +
"\nPreserve Expiry: true" +
"}"
if !strings.EqualFold(expected, toStr) {
t.Fatalf("Expected packet string value of \n%s\n did not match actual \n%s", expected, toStr)
}
}
gocbcore-10.2.3/memd/srvdura16.go 0000664 0000000 0000000 00000001325 14417540156 0016476 0 ustar 00root root 0000000 0000000 package memd
import (
"math"
"time"
)
// EncodeSrvDura16 takes a standard go time duration and encodes it into
// the appropriate format for the server.
func EncodeSrvDura16(dura time.Duration) uint16 {
serverDurationUs := dura / time.Microsecond
serverDurationEnc := int(math.Pow(float64(serverDurationUs)*2, 1.0/1.74))
if serverDurationEnc > 65535 {
serverDurationEnc = 65535
}
return uint16(serverDurationEnc)
}
// DecodeSrvDura16 takes an encoded operation duration from the server
// and converts it to a standard Go time duration.
func DecodeSrvDura16(enc uint16) time.Duration {
serverDurationUs := math.Round(math.Pow(float64(enc), 1.74) / 2)
return time.Duration(serverDurationUs) * time.Microsecond
}
gocbcore-10.2.3/memd/srvdura16_test.go 0000664 0000000 0000000 00000001234 14417540156 0017534 0 ustar 00root root 0000000 0000000 package memd
import (
"testing"
"time"
)
func testSrvDura16(t *testing.T, n time.Duration, e uint16) {
t.Helper()
x := EncodeSrvDura16(n)
if x != e {
t.Fatalf("encoding failed %d != %d", x, e)
}
y := DecodeSrvDura16(e)
if y != n {
t.Fatalf("decoding failed %d != %d", y, n)
}
}
func TestSrvDura16(t *testing.T) {
// Note that these values are specifically selected as they are
// known to be encoded exactly with eachother (which is what
// we do our testing to)
testSrvDura16(t, 0*time.Microsecond, 0)
testSrvDura16(t, 1331*time.Microsecond, 93)
testSrvDura16(t, 20841*time.Microsecond, 452)
testSrvDura16(t, 119973*time.Microsecond, 1236)
}
gocbcore-10.2.3/memd/statuscode.go 0000664 0000000 0000000 00000032513 14417540156 0017022 0 ustar 00root root 0000000 0000000 package memd
import "fmt"
// StatusCode represents a memcached response status.
type StatusCode uint16
const (
// StatusSuccess indicates the operation completed successfully.
StatusSuccess = StatusCode(0x00)
// StatusKeyNotFound occurs when an operation is performed on a key that does not exist.
StatusKeyNotFound = StatusCode(0x01)
// StatusKeyExists occurs when an operation is performed on a key that could not be found.
StatusKeyExists = StatusCode(0x02)
// StatusTooBig occurs when an operation attempts to store more data in a single document
// than the server is capable of storing (by default, this is a 20MB limit).
StatusTooBig = StatusCode(0x03)
// StatusInvalidArgs occurs when the server receives invalid arguments for an operation.
StatusInvalidArgs = StatusCode(0x04)
// StatusNotStored occurs when the server fails to store a key.
StatusNotStored = StatusCode(0x05)
// StatusBadDelta occurs when an invalid delta value is specified to a counter operation.
StatusBadDelta = StatusCode(0x06)
// StatusNotMyVBucket occurs when an operation is dispatched to a server which is
// non-authoritative for a specific vbucket.
StatusNotMyVBucket = StatusCode(0x07)
// StatusNoBucket occurs when no bucket was selected on a connection.
StatusNoBucket = StatusCode(0x08)
// StatusLocked occurs when an operation fails due to the document being locked.
StatusLocked = StatusCode(0x09)
// StatusAuthStale occurs when authentication credentials have become invalidated.
StatusAuthStale = StatusCode(0x1f)
// StatusAuthError occurs when the authentication information provided was not valid.
StatusAuthError = StatusCode(0x20)
// StatusAuthContinue occurs in multi-step authentication when more authentication
// work needs to be performed in order to complete the authentication process.
StatusAuthContinue = StatusCode(0x21)
// StatusRangeError occurs when the range specified to the server is not valid.
StatusRangeError = StatusCode(0x22)
// StatusRollback occurs when a DCP stream fails to open due to a rollback having
// previously occurred since the last time the stream was opened.
StatusRollback = StatusCode(0x23)
// StatusAccessError occurs when an access error occurs.
StatusAccessError = StatusCode(0x24)
// StatusNotInitialized is sent by servers which are still initializing, and are not
// yet ready to accept operations on behalf of a particular bucket.
StatusNotInitialized = StatusCode(0x25)
// StatusRateLimitedNetworkIngress occurs when the server rate limits due to network ingress.
StatusRateLimitedNetworkIngress = StatusCode(0x30)
// StatusRateLimitedNetworkEgress occurs when the server rate limits due to network egress.
StatusRateLimitedNetworkEgress = StatusCode(0x31)
// StatusRateLimitedMaxConnections occurs when the server rate limits due to the application reaching the maximum
// number of allowed connections.
StatusRateLimitedMaxConnections = StatusCode(0x32)
// StatusRateLimitedMaxCommands occurs when the server rate limits due to the application reaching the maximum
// number of allowed operations.
StatusRateLimitedMaxCommands = StatusCode(0x33)
// StatusRateLimitedScopeSizeLimitExceeded occurs when the server rate limits due to the application reaching the maximum
// data size allowed for the scope.
StatusRateLimitedScopeSizeLimitExceeded = StatusCode(0x34)
// StatusUnknownCommand occurs when an unknown operation is sent to a server.
StatusUnknownCommand = StatusCode(0x81)
// StatusOutOfMemory occurs when the server cannot service a request due to memory
// limitations.
StatusOutOfMemory = StatusCode(0x82)
// StatusNotSupported occurs when an operation is understood by the server, but that
// operation is not supported on this server (occurs for a variety of reasons).
StatusNotSupported = StatusCode(0x83)
// StatusInternalError occurs when internal errors prevent the server from processing
// your request.
StatusInternalError = StatusCode(0x84)
// StatusBusy occurs when the server is too busy to process your request right away.
// Attempting the operation at a later time will likely succeed.
StatusBusy = StatusCode(0x85)
// StatusTmpFail occurs when a temporary failure is preventing the server from
// processing your request.
StatusTmpFail = StatusCode(0x86)
// StatusCollectionUnknown occurs when a Collection cannot be found.
StatusCollectionUnknown = StatusCode(0x88)
// StatusScopeUnknown occurs when a Scope cannot be found.
StatusScopeUnknown = StatusCode(0x8c)
// StatusDCPStreamIDInvalid occurs when a dcp stream ID is invalid.
StatusDCPStreamIDInvalid = StatusCode(0x8d)
// StatusDurabilityInvalidLevel occurs when an invalid durability level was requested.
StatusDurabilityInvalidLevel = StatusCode(0xa0)
// StatusDurabilityImpossible occurs when a request is performed with impossible
// durability level requirements.
StatusDurabilityImpossible = StatusCode(0xa1)
// StatusSyncWriteInProgress occurs when an attempt is made to write to a key that has
// a SyncWrite pending.
StatusSyncWriteInProgress = StatusCode(0xa2)
// StatusSyncWriteAmbiguous occurs when an SyncWrite does not complete in the specified
// time and the result is ambiguous.
StatusSyncWriteAmbiguous = StatusCode(0xa3)
// StatusSyncWriteReCommitInProgress occurs when an SyncWrite is being recommitted.
StatusSyncWriteReCommitInProgress = StatusCode(0xa4)
// StatusRangeScanCancelled occurs during a range scan to indicate that the range scan was cancelled.
StatusRangeScanCancelled = StatusCode(0xa5)
// StatusRangeScanMore occurs during a range scan to indicate that a range scan has more results.
StatusRangeScanMore = StatusCode(0xa6)
// StatusRangeScanComplete occurs during a range scan to indicate that a range scan has completed.
StatusRangeScanComplete = StatusCode(0xa7)
// StatusRangeScanVbUUIDNotEqual occurs during a range scan to indicate that a vb-uuid mismatch has occurred.
StatusRangeScanVbUUIDNotEqual = StatusCode(0xa8)
// StatusSubDocPathNotFound occurs when a sub-document operation targets a path
// which does not exist in the specifie document.
StatusSubDocPathNotFound = StatusCode(0xc0)
// StatusSubDocPathMismatch occurs when a sub-document operation specifies a path
// which does not match the document structure (field access on an array).
StatusSubDocPathMismatch = StatusCode(0xc1)
// StatusSubDocPathInvalid occurs when a sub-document path could not be parsed.
StatusSubDocPathInvalid = StatusCode(0xc2)
// StatusSubDocPathTooBig occurs when a sub-document path is too big.
StatusSubDocPathTooBig = StatusCode(0xc3)
// StatusSubDocDocTooDeep occurs when an operation would cause a document to be
// nested beyond the depth limits allowed by the sub-document specification.
StatusSubDocDocTooDeep = StatusCode(0xc4)
// StatusSubDocCantInsert occurs when a sub-document operation could not insert.
StatusSubDocCantInsert = StatusCode(0xc5)
// StatusSubDocNotJSON occurs when a sub-document operation is performed on a
// document which is not JSON.
StatusSubDocNotJSON = StatusCode(0xc6)
// StatusSubDocBadRange occurs when a sub-document operation is performed with
// a bad range.
StatusSubDocBadRange = StatusCode(0xc7)
// StatusSubDocBadDelta occurs when a sub-document counter operation is performed
// and the specified delta is not valid.
StatusSubDocBadDelta = StatusCode(0xc8)
// StatusSubDocPathExists occurs when a sub-document operation expects a path not
// to exists, but the path was found in the document.
StatusSubDocPathExists = StatusCode(0xc9)
// StatusSubDocValueTooDeep occurs when a sub-document operation specifies a value
// which is deeper than the depth limits of the sub-document specification.
StatusSubDocValueTooDeep = StatusCode(0xca)
// StatusSubDocBadCombo occurs when a multi-operation sub-document operation is
// performed and operations within the package of ops conflict with each other.
StatusSubDocBadCombo = StatusCode(0xcb)
// StatusSubDocBadMulti occurs when a multi-operation sub-document operation is
// performed and operations within the package of ops conflict with each other.
StatusSubDocBadMulti = StatusCode(0xcc)
// StatusSubDocSuccessDeleted occurs when a multi-operation sub-document operation
// is performed on a soft-deleted document.
StatusSubDocSuccessDeleted = StatusCode(0xcd)
// StatusSubDocXattrInvalidFlagCombo occurs when an invalid set of
// extended-attribute flags is passed to a sub-document operation.
StatusSubDocXattrInvalidFlagCombo = StatusCode(0xce)
// StatusSubDocXattrInvalidKeyCombo occurs when an invalid set of key operations
// are specified for a extended-attribute sub-document operation.
StatusSubDocXattrInvalidKeyCombo = StatusCode(0xcf)
// StatusSubDocXattrUnknownMacro occurs when an invalid macro value is specified.
StatusSubDocXattrUnknownMacro = StatusCode(0xd0)
// StatusSubDocXattrUnknownVAttr occurs when an invalid virtual attribute is specified.
StatusSubDocXattrUnknownVAttr = StatusCode(0xd1)
// StatusSubDocXattrCannotModifyVAttr occurs when a mutation is attempted upon
// a virtual attribute (which are immutable by definition).
StatusSubDocXattrCannotModifyVAttr = StatusCode(0xd2)
// StatusSubDocMultiPathFailureDeleted occurs when a Multi Path Failure occurs on
// a soft-deleted document.
StatusSubDocMultiPathFailureDeleted = StatusCode(0xd3)
)
// String returns the textual representation of this StatusCode.
func (code StatusCode) String() string {
switch code {
case StatusSuccess:
return "success"
case StatusKeyNotFound:
return "key not found"
case StatusKeyExists:
return "key already exists, if a cas was provided the key exists with a different cas"
case StatusTooBig:
return "document value was too large"
case StatusInvalidArgs:
return "invalid arguments"
case StatusNotStored:
return "document could not be stored"
case StatusBadDelta:
return "invalid delta was passed"
case StatusNotMyVBucket:
return "operation sent to incorrect server"
case StatusNoBucket:
return "not connected to a bucket"
case StatusAuthStale:
return "authentication context is stale, try re-authenticating"
case StatusAuthError:
return "authentication error"
case StatusAuthContinue:
return "more authentication steps needed"
case StatusRangeError:
return "requested value is outside range"
case StatusAccessError:
return "no access"
case StatusNotInitialized:
return "cluster is being initialized, requests are blocked"
case StatusRollback:
return "rollback is required"
case StatusUnknownCommand:
return "unknown command was received"
case StatusOutOfMemory:
return "server is out of memory"
case StatusNotSupported:
return "server does not support this command"
case StatusInternalError:
return "internal server error"
case StatusBusy:
return "server is busy, try again later"
case StatusTmpFail:
return "temporary failure occurred, try again later"
case StatusCollectionUnknown:
return "the requested collection cannot be found"
case StatusScopeUnknown:
return "the requested scope cannot be found."
case StatusDCPStreamIDInvalid:
return "the provided stream ID is invalid"
case StatusDurabilityInvalidLevel:
return "invalid request, invalid durability level specified."
case StatusDurabilityImpossible:
return "the requested durability requirements are impossible."
case StatusSyncWriteInProgress:
return "key already has syncwrite pending."
case StatusSyncWriteAmbiguous:
return "the syncwrite request did not complete in time."
case StatusSubDocPathNotFound:
return "sub-document path does not exist"
case StatusSubDocPathMismatch:
return "type of element in sub-document path conflicts with type in document"
case StatusSubDocPathInvalid:
return "malformed sub-document path"
case StatusSubDocPathTooBig:
return "sub-document contains too many components"
case StatusSubDocDocTooDeep:
return "existing document contains too many levels of nesting"
case StatusSubDocCantInsert:
return "subdocument operation would invalidate the JSON"
case StatusSubDocNotJSON:
return "existing document is not valid JSON"
case StatusSubDocBadRange:
return "existing numeric value is too large"
case StatusSubDocBadDelta:
return "numeric operation would yield a number that is too large, or " +
"a zero delta was specified"
case StatusSubDocPathExists:
return "given path already exists in the document"
case StatusSubDocValueTooDeep:
return "value is too deep to insert"
case StatusSubDocBadCombo:
return "incorrectly matched subdocument operation types"
case StatusSubDocBadMulti:
return "could not execute one or more multi lookups or mutations"
case StatusSubDocSuccessDeleted:
return "document is soft-deleted"
case StatusSubDocXattrInvalidFlagCombo:
return "invalid xattr flag combination"
case StatusSubDocXattrInvalidKeyCombo:
return "invalid xattr key combination"
case StatusSubDocXattrUnknownMacro:
return "unknown xattr macro"
case StatusSubDocXattrUnknownVAttr:
return "unknown xattr virtual attribute"
case StatusSubDocXattrCannotModifyVAttr:
return "cannot modify virtual attributes"
case StatusSubDocMultiPathFailureDeleted:
return "sub-document multi-path error"
case StatusRangeScanCancelled:
return "range scan cancelled"
case StatusRangeScanComplete:
return "range scan complete"
case StatusRangeScanMore:
return "range scan more"
case StatusRangeScanVbUUIDNotEqual:
return "range scan vb-uuid not equal"
default:
return fmt.Sprintf("unknown kv status code (%d)", code)
}
}
gocbcore-10.2.3/memd/uleb128.go 0000664 0000000 0000000 00000001665 14417540156 0016032 0 ustar 00root root 0000000 0000000 package memd
import (
"errors"
)
// AppendULEB128_32 appends a 32-bit number encoded as ULEB128 to a byte slice
func AppendULEB128_32(b []byte, v uint32) []byte {
for {
c := uint8(v & 0x7f)
v >>= 7
if v != 0 {
c |= 0x80
}
b = append(b, c)
if c&0x80 == 0 {
break
}
}
return b
}
// DecodeULEB128_32 decodes a ULEB128 encoded number into a uint32
func DecodeULEB128_32(b []byte) (uint32, int, error) {
if len(b) == 0 {
return 0, 0, errors.New("no data provided")
}
var u uint64
var n int
for i := 0; ; i++ {
if i >= len(b) {
return 0, 0, errors.New("encoded number is longer than provided data")
}
if i*7 > 32 {
// oversize and then break to get caught below
u = 0xffffffffffffffff
break
}
u |= uint64(b[i]&0x7f) << (i * 7)
if b[i]&0x80 == 0 {
n = i + 1
break
}
}
if u > 0xffffffff {
return 0, 0, errors.New("encoded data is longer than 32 bits")
}
return uint32(u), n, nil
}
gocbcore-10.2.3/memd/uleb128_test.go 0000664 0000000 0000000 00000004063 14417540156 0017064 0 ustar 00root root 0000000 0000000 package memd
import (
"bytes"
"testing"
)
func testULEB128_32(t *testing.T, v uint32, eb []byte) {
t.Helper()
buf := AppendULEB128_32(nil, v)
bufLen := len(buf)
if bytes.Compare(buf, eb) != 0 {
t.Fatalf("failed to encode: %+v != %+v", buf, eb)
}
// add some garbage to the end for fun
buf = append(buf, 0xFF, 0x88, 0x00)
x, n, err := DecodeULEB128_32(buf)
if err != nil {
t.Fatalf("failed to decode: %s", err)
}
if n != bufLen {
t.Fatalf("wrong number of decoded bytes")
}
if x != v {
t.Fatalf("wrong decoded value: %d != %d", x, v)
}
}
func TestULEB128_32_0x00000000(t *testing.T) {
testULEB128_32(t, 0x00000000, []byte{0x00})
}
func TestULEB128_32_0x00000001(t *testing.T) {
testULEB128_32(t, 0x00000001, []byte{0x01})
}
func TestULEB128_32_0x0000007F(t *testing.T) {
testULEB128_32(t, 0x0000007F, []byte{0x7F})
}
func TestULEB128_32_0x00000080(t *testing.T) {
testULEB128_32(t, 0x00000080, []byte{0x80, 0x01})
}
func TestULEB128_32_0x00000555(t *testing.T) {
testULEB128_32(t, 0x00000555, []byte{0xD5, 0x0A})
}
func TestULEB128_32_0x00007FFF(t *testing.T) {
testULEB128_32(t, 0x00007FFF, []byte{0xFF, 0xFF, 0x01})
}
func TestULEB128_32_0x0000BFFF(t *testing.T) {
testULEB128_32(t, 0x0000BFFF, []byte{0xFF, 0xFF, 0x02})
}
func TestULEB128_32_0x0000FFFF(t *testing.T) {
testULEB128_32(t, 0x0000FFFF, []byte{0xFF, 0xFF, 0x03})
}
func TestULEB128_32_0x00008000(t *testing.T) {
testULEB128_32(t, 0x00008000, []byte{0x80, 0x80, 0x02})
}
func TestULEB128_32_0x00005555(t *testing.T) {
testULEB128_32(t, 0x00005555, []byte{0xD5, 0xAA, 0x01})
}
func TestULEB128_32_0x0CAFEF00(t *testing.T) {
testULEB128_32(t, 0x0CAFEF00, []byte{0x80, 0xDE, 0xBF, 0x65})
}
func TestULEB128_32_0xCAFEF00D(t *testing.T) {
testULEB128_32(t, 0xCAFEF00D, []byte{0x8D, 0xE0, 0xFB, 0xD7, 0x0C})
}
func TestULEB128_32_0xffffffff(t *testing.T) {
testULEB128_32(t, 0xffffffff, []byte{0xFF, 0xFF, 0xFF, 0xFF, 0x0F})
}
func TestULEB128_32_nil(t *testing.T) {
_, _, err := DecodeULEB128_32([]byte{})
if err == nil {
t.Fatal("decoding should have failed but did not")
}
}
gocbcore-10.2.3/memdbootstrap_client.go 0000664 0000000 0000000 00000022274 14417540156 0020143 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/binary"
"errors"
"strings"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
type bootstrapableClient interface {
SendRequest(*memdQRequest) error
Address() string
ConnID() string
SupportsFeature(feature memd.HelloFeature) bool
Features([]memd.HelloFeature)
}
type bootstrapClient interface {
Address() string
ConnID() string
Features(features []memd.HelloFeature)
SupportsFeature(feature memd.HelloFeature) bool
SaslAuth(k, v []byte, deadline time.Time, cb func(b []byte, err error)) error
SaslStep(k, v []byte, deadline time.Time, cb func(err error)) error
ExecSelectBucket(b []byte, deadline time.Time) (chan error, error)
ExecGetErrorMap(version uint16, deadline time.Time) (chan errorMapResponse, error)
SaslListMechs(deadline time.Time, cb func(mechs []AuthMechanism, err error)) error
ExecHello(clientID string, features []memd.HelloFeature, deadline time.Time) (chan ExecHelloResponse, error)
ExecGetConfig(deadline time.Time) (chan getConfigResponse, error)
}
// Due to AuthProvider we are currently tied to bootstrapping passing around a deadline and the bootstrap
// "owner" has to hold onto a cancel sig for use at request time.
// In the future we can combine deadline and cancellation into a context.Context and pass that everywhere as a parameter,
// we will then able to expose utility functions to allow user to build their own bootstrap from existing building
// blocks.
func newMemdBootstrapClient(client bootstrapableClient, cancelSig <-chan struct{}) *memdBootstrapClient {
return &memdBootstrapClient{
cancelSig: cancelSig,
client: client,
}
}
type memdBootstrapClient struct {
client bootstrapableClient
cancelSig <-chan struct{}
}
func (bc *memdBootstrapClient) Address() string {
return bc.client.Address()
}
func (bc *memdBootstrapClient) ConnID() string {
return bc.client.ConnID()
}
func (bc *memdBootstrapClient) Features(features []memd.HelloFeature) {
bc.client.Features(features)
}
func (bc *memdBootstrapClient) SupportsFeature(feature memd.HelloFeature) bool {
return bc.client.SupportsFeature(feature)
}
func (bc *memdBootstrapClient) SaslAuth(k, v []byte, deadline time.Time, cb func(b []byte, err error)) error {
err := bc.doBootstrapRequest(
&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdSASLAuth,
Key: k,
Value: v,
},
Callback: func(resp *memdQResponse, _ *memdQRequest, err error) {
// Auth is special, auth continue is surfaced as an error
var val []byte
if resp != nil {
val = resp.Value
}
cb(val, err)
},
RetryStrategy: newFailFastRetryStrategy(),
},
deadline,
)
if err != nil {
return err
}
return nil
}
func (bc *memdBootstrapClient) SaslStep(k, v []byte, deadline time.Time, cb func(err error)) error {
err := bc.doBootstrapRequest(
&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdSASLStep,
Key: k,
Value: v,
},
Callback: func(resp *memdQResponse, _ *memdQRequest, err error) {
if err != nil {
cb(err)
return
}
cb(nil)
},
RetryStrategy: newFailFastRetryStrategy(),
},
deadline,
)
if err != nil {
return err
}
return nil
}
func (bc *memdBootstrapClient) ExecSelectBucket(b []byte, deadline time.Time) (chan error, error) {
completedCh := make(chan error, 1)
err := bc.doBootstrapRequest(
&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdSelectBucket,
Key: b,
},
Callback: func(resp *memdQResponse, _ *memdQRequest, err error) {
if err != nil {
if errors.Is(err, ErrDocumentNotFound) {
// Bucket not found means that the user has privileges to access the bucket but that the bucket
// is in some way not existing right now (e.g. in warmup).
err = errBucketNotFound
}
completedCh <- err
return
}
completedCh <- nil
},
RetryStrategy: newFailFastRetryStrategy(),
},
deadline,
)
if err != nil {
return nil, err
}
return completedCh, nil
}
type errorMapResponse struct {
Err error
Bytes []byte
}
func (bc *memdBootstrapClient) ExecGetErrorMap(version uint16, deadline time.Time) (chan errorMapResponse, error) {
completedCh := make(chan errorMapResponse, 1)
valueBuf := make([]byte, 2)
binary.BigEndian.PutUint16(valueBuf, version)
err := bc.doBootstrapRequest(
&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGetErrorMap,
Value: valueBuf,
},
Callback: func(resp *memdQResponse, _ *memdQRequest, err error) {
if err != nil {
completedCh <- errorMapResponse{
Err: err,
}
return
}
completedCh <- errorMapResponse{
Bytes: resp.Value,
}
},
RetryStrategy: newFailFastRetryStrategy(),
},
deadline,
)
if err != nil {
return nil, err
}
return completedCh, nil
}
func (bc *memdBootstrapClient) SaslListMechs(deadline time.Time, cb func(mechs []AuthMechanism, err error)) error {
err := bc.doBootstrapRequest(
&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdSASLListMechs,
},
Callback: func(resp *memdQResponse, _ *memdQRequest, err error) {
if err != nil {
cb(nil, err)
return
}
mechs := strings.Split(string(resp.Value), " ")
var authMechs []AuthMechanism
for _, mech := range mechs {
authMechs = append(authMechs, AuthMechanism(mech))
}
cb(authMechs, nil)
},
RetryStrategy: newFailFastRetryStrategy(),
},
deadline,
)
if err != nil {
return err
}
return nil
}
// ExecHelloResponse contains the features and/or error from an ExecHello operation.
type ExecHelloResponse struct {
SrvFeatures []memd.HelloFeature
Err error
}
func (bc *memdBootstrapClient) ExecHello(clientID string, features []memd.HelloFeature, deadline time.Time) (chan ExecHelloResponse, error) {
appendFeatureCode := func(bytes []byte, feature memd.HelloFeature) []byte {
bytes = append(bytes, 0, 0)
binary.BigEndian.PutUint16(bytes[len(bytes)-2:], uint16(feature))
return bytes
}
var featureBytes []byte
for _, feature := range features {
featureBytes = appendFeatureCode(featureBytes, feature)
}
completedCh := make(chan ExecHelloResponse, 1)
err := bc.doBootstrapRequest(
&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdHello,
Key: []byte(clientID),
Value: featureBytes,
},
Callback: func(resp *memdQResponse, _ *memdQRequest, err error) {
if err != nil {
completedCh <- ExecHelloResponse{
Err: err,
}
return
}
var srvFeatures []memd.HelloFeature
for i := 0; i < len(resp.Value); i += 2 {
feature := binary.BigEndian.Uint16(resp.Value[i:])
srvFeatures = append(srvFeatures, memd.HelloFeature(feature))
}
completedCh <- ExecHelloResponse{
SrvFeatures: srvFeatures,
}
},
RetryStrategy: newFailFastRetryStrategy(),
},
deadline,
)
if err != nil {
return nil, err
}
return completedCh, nil
}
type getConfigResponse struct {
Err error
Config *cfgBucket
}
func (bc *memdBootstrapClient) ExecGetConfig(deadline time.Time) (chan getConfigResponse, error) {
completedCh := make(chan getConfigResponse, 1)
err := bc.doBootstrapRequest(
&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdGetClusterConfig,
},
Callback: func(resp *memdQResponse, _ *memdQRequest, err error) {
if err != nil {
completedCh <- getConfigResponse{
Err: err,
}
return
}
hostName, err := hostFromHostPort(bc.Address())
if err != nil {
logWarnf("Boostrap client: Failed to parse source address. %s", err)
completedCh <- getConfigResponse{
Err: err,
}
return
}
bk, err := parseConfig(resp.Value, hostName)
if err != nil {
logWarnf("Boostrap client: Failed to parse CCCP config. %v", err)
completedCh <- getConfigResponse{
Err: err,
}
return
}
completedCh <- getConfigResponse{
Config: bk,
}
},
RetryStrategy: newFailFastRetryStrategy(),
},
deadline,
)
if err != nil {
return nil, err
}
return completedCh, nil
}
func (bc *memdBootstrapClient) doBootstrapRequest(req *memdQRequest, deadline time.Time) error {
origCb := req.Callback
doneCh := make(chan struct{})
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
close(doneCh)
origCb(resp, req, err)
}
req.Callback = handler
err := bc.client.SendRequest(req)
if err != nil {
return err
}
start := time.Now()
req.SetTimer(time.AfterFunc(deadline.Sub(start), func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallback(&TimeoutError{
InnerError: errAmbiguousTimeout,
OperationID: req.Command.Name(),
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
})
}))
go func() {
select {
case <-doneCh:
return
case <-bc.cancelSig:
req.Cancel()
<-doneCh
return
}
}()
return nil
}
gocbcore-10.2.3/memdbootstrap_dcp_client.go 0000664 0000000 0000000 00000005663 14417540156 0020774 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/binary"
"fmt"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func newDCPBootstrapClient(client *memdBootstrapClient) *dcpBootstrapClient {
return &dcpBootstrapClient{
memdBootstrapClient: client,
}
}
type dcpBootstrapClient struct {
*memdBootstrapClient
}
func (client *dcpBootstrapClient) ExecDcpControl(key string, value string, deadline time.Time) error {
_, err := client.sendRequest(memd.CmdDcpControl, []byte(key), []byte(value), nil, deadline)
return err
}
func (client *dcpBootstrapClient) ExecGetClusterConfig(deadline time.Time) ([]byte, error) {
return client.sendRequest(memd.CmdGetClusterConfig, nil, nil, nil, deadline)
}
func (client *dcpBootstrapClient) ExecOpenDcpConsumer(streamName string, openFlags memd.DcpOpenFlag, deadline time.Time) error {
extraBuf := make([]byte, 8)
binary.BigEndian.PutUint32(extraBuf[0:], 0)
binary.BigEndian.PutUint32(extraBuf[4:], uint32((openFlags & ^memd.DcpOpenFlag(3))|memd.DcpOpenFlagProducer))
_, err := client.sendRequest(memd.CmdDcpOpenConnection, []byte(streamName), nil, extraBuf, deadline)
return err
}
func (client *dcpBootstrapClient) ExecEnableDcpNoop(period time.Duration, deadline time.Time) error {
// The client will always reply to No-Op's. No need to enable it
err := client.ExecDcpControl("enable_noop", "true", deadline)
if err != nil {
return err
}
periodStr := fmt.Sprintf("%d", period/time.Second)
err = client.ExecDcpControl("set_noop_interval", periodStr, deadline)
if err != nil {
return err
}
return nil
}
func (client *dcpBootstrapClient) ExecEnableDcpClientEnd(deadline time.Time) error {
memcli, ok := client.client.(*memdClient)
if !ok {
return errCliInternalError
}
err := client.ExecDcpControl("send_stream_end_on_client_close_stream", "true", deadline)
if err != nil {
memcli.streamEndNotSupported = true
}
return nil
}
func (client *dcpBootstrapClient) ExecEnableDcpBufferAck(bufferSize int, deadline time.Time) error {
mclient, ok := client.client.(*memdClient)
if !ok {
return errCliInternalError
}
// Enable buffer acknowledgment on the client
mclient.EnableDcpBufferAck(bufferSize / 2)
bufferSizeStr := fmt.Sprintf("%d", bufferSize)
err := client.ExecDcpControl("connection_buffer_size", bufferSizeStr, deadline)
if err != nil {
return err
}
return nil
}
func (bc *memdBootstrapClient) sendRequest(cmd memd.CmdCode, k, v, e []byte, deadline time.Time) (valOut []byte, errOut error) {
signal := make(chan struct{}, 1)
err := bc.doBootstrapRequest(&memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: cmd,
Key: k,
Value: v,
Extras: e,
},
Callback: func(resp *memdQResponse, _ *memdQRequest, err error) {
if resp != nil {
valOut = resp.Packet.Value
}
errOut = err
signal <- struct{}{}
},
RetryStrategy: newFailFastRetryStrategy(),
}, deadline)
if err != nil {
if err != nil {
return nil, err
}
}
<-signal
return
}
gocbcore-10.2.3/memdclient.go 0000664 0000000 0000000 00000046406 14417540156 0016051 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/binary"
"fmt"
"io"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/couchbase/gocbcore/v10/memd"
"github.com/golang/snappy"
)
func isCompressibleOp(command memd.CmdCode) bool {
switch command {
case memd.CmdSet:
fallthrough
case memd.CmdAdd:
fallthrough
case memd.CmdReplace:
fallthrough
case memd.CmdAppend:
fallthrough
case memd.CmdPrepend:
return true
}
return false
}
type postCompleteErrorHandler func(resp *memdQResponse, req *memdQRequest, err error) (bool, error)
type memdClient struct {
lastActivity int64
dcpAckSize int
dcpFlowRecv int
closeNotify chan bool
connReleaseNotify chan struct{}
connReleasedNotify chan struct{}
connID string
closed bool
conn memdConn
opList *memdOpMap
features []memd.HelloFeature
lock sync.Mutex
streamEndNotSupported bool
breaker circuitBreaker
postErrHandler postCompleteErrorHandler
tracer *tracerComponent
zombieLogger *zombieLoggerComponent
dcpQueueSize int
// When a close request comes in, we need to immediately stop processing all requests. This
// includes immediately stopping the DCP queue rather than waiting for the application to
// flush that queue. This means that we lose packets that were read but not processed, but
// this is not fundamentally different to if we had just not read them at all. As a side
// effect of this, we need to use a separate kill signal on top of closing the queue.
// We need this to be owned by the client because we only use it when the client is closed,
// when the connection is closed from an external actor (e.g. server) we want to flush the queue.
shutdownDCP uint32
compressionMinSize int
compressionMinRatio float64
disableDecompression bool
gracefulCloseTriggered uint32
}
type dcpBuffer struct {
resp *memdQResponse
packetLen int
isInternal bool
}
type memdClientProps struct {
ClientID string
DCPQueueSize int
CompressionMinSize int
CompressionMinRatio float64
DisableDecompression bool
}
func newMemdClient(props memdClientProps, conn memdConn, breakerCfg CircuitBreakerConfig, postErrHandler postCompleteErrorHandler,
tracer *tracerComponent, zombieLogger *zombieLoggerComponent) *memdClient {
client := memdClient{
closeNotify: make(chan bool),
connReleaseNotify: make(chan struct{}),
connReleasedNotify: make(chan struct{}),
connID: props.ClientID + "/" + formatCbUID(randomCbUID()),
postErrHandler: postErrHandler,
tracer: tracer,
zombieLogger: zombieLogger,
conn: conn,
opList: newMemdOpMap(),
dcpQueueSize: props.DCPQueueSize,
compressionMinRatio: props.CompressionMinRatio,
compressionMinSize: props.CompressionMinSize,
disableDecompression: props.DisableDecompression,
}
if breakerCfg.Enabled {
client.breaker = newLazyCircuitBreaker(breakerCfg, client.sendCanary)
} else {
client.breaker = newNoopCircuitBreaker()
}
client.run()
return &client
}
func (client *memdClient) SupportsFeature(feature memd.HelloFeature) bool {
return checkSupportsFeature(client.features, feature)
}
// Features must be set from a context where no racey behaviours can occur, i.e. during bootstrap.
func (client *memdClient) Features(features []memd.HelloFeature) {
client.features = features
for _, feature := range features {
client.conn.EnableFeature(feature)
}
}
func (client *memdClient) EnableDcpBufferAck(bufferAckSize int) {
client.dcpAckSize = bufferAckSize
}
func (client *memdClient) maybeSendDcpBufferAck(packetLen int) {
client.dcpFlowRecv += packetLen
if client.dcpFlowRecv < client.dcpAckSize {
return
}
ackAmt := client.dcpFlowRecv
extrasBuf := make([]byte, 4)
binary.BigEndian.PutUint32(extrasBuf, uint32(ackAmt))
err := client.conn.WritePacket(&memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdDcpBufferAck,
Extras: extrasBuf,
})
if err != nil {
logWarnf("%p memdclient failed to dispatch DCP buffer ack: %s", client, err)
}
client.dcpFlowRecv -= ackAmt
}
func (client *memdClient) Address() string {
return client.conn.RemoteAddr()
}
func (client *memdClient) ConnID() string {
return client.connID
}
func (client *memdClient) CloseNotify() chan bool {
return client.closeNotify
}
func (client *memdClient) takeRequestOwnership(req *memdQRequest) error {
client.lock.Lock()
defer client.lock.Unlock()
if client.closed {
logDebugf("%s memdclient attempted to put dispatched op OP=0x%x, Opaque=%d in drained opmap", client.loggerID(), req.Command, req.Opaque)
return errMemdClientClosed
}
if atomic.LoadUint32(&client.gracefulCloseTriggered) == 1 {
logDebugf("%s memdclient attempted to dispatch op OP=0x%x, Opaque=%d from gracefully closing memdclient", client.loggerID(), req.Command, req.Opaque)
return errMemdClientClosed
}
if !atomic.CompareAndSwapPointer(&req.waitingIn, nil, unsafe.Pointer(client)) {
logDebugf("%s memdclient attempted to put dispatched op OP=0x%x, Opaque=%d in new opmap", client.loggerID(), req.Command, req.Opaque)
return errRequestAlreadyDispatched
}
if req.isCancelled() {
atomic.CompareAndSwapPointer(&req.waitingIn, unsafe.Pointer(client), nil)
return errRequestCanceled
}
connInfo := memdQRequestConnInfo{
lastDispatchedTo: client.Address(),
lastDispatchedFrom: client.conn.LocalAddr(),
lastConnectionID: client.connID,
}
req.SetConnectionInfo(connInfo)
client.opList.Add(req)
return nil
}
func (client *memdClient) CancelRequest(req *memdQRequest, err error) bool {
client.lock.Lock()
defer client.lock.Unlock()
if client.closed {
logDebugf("%s memdclient attempted to remove op OP=0x%x, Opaque=%d from drained opmap", client.loggerID(), req.Command, req.Opaque)
return false
}
removed := client.opList.Remove(req)
if removed {
atomic.CompareAndSwapPointer(&req.waitingIn, unsafe.Pointer(client), nil)
}
if client.breaker.CompletionCallback(err) {
client.breaker.MarkSuccessful()
} else {
client.breaker.MarkFailure()
}
return removed
}
func (client *memdClient) SendRequest(req *memdQRequest) error {
if !client.breaker.AllowsRequest() {
logSchedf("Circuit breaker interrupting request. %s to %s OP=0x%x. Opaque=%d", client.conn.LocalAddr(), client.Address(), req.Command, req.Opaque)
req.cancelWithCallback(errCircuitBreakerOpen)
return nil
}
return client.internalSendRequest(req)
}
func (client *memdClient) internalSendRequest(req *memdQRequest) error {
if err := client.takeRequestOwnership(req); err != nil {
return err
}
packet := &req.Packet
if client.SupportsFeature(memd.FeatureSnappy) {
isCompressed := (packet.Datatype & uint8(memd.DatatypeFlagCompressed)) != 0
packetSize := len(packet.Value)
if !isCompressed && packetSize > client.compressionMinSize && isCompressibleOp(packet.Command) {
compressedValue := snappy.Encode(nil, packet.Value)
if float64(len(compressedValue))/float64(packetSize) <= client.compressionMinRatio {
newPacket := *packet
newPacket.Value = compressedValue
newPacket.Datatype = newPacket.Datatype | uint8(memd.DatatypeFlagCompressed)
packet = &newPacket
}
}
}
logSchedf("Writing request. %s to %s OP=0x%x. Opaque=%d", client.conn.LocalAddr(), client.Address(), req.Command, req.Opaque)
client.tracer.StartNetTrace(req)
err := client.conn.WritePacket(packet)
if err != nil {
logDebugf(" %s memdclient write failure: %v", client.loggerID(), err)
return err
}
return nil
}
func (client *memdClient) classifyResponseStatusClass(status memd.StatusCode) statusClass {
switch status {
case memd.StatusSuccess:
return statusClassOK
case memd.StatusRangeScanMore:
return statusClassOK
case memd.StatusRangeScanComplete:
return statusClassOK
default:
return statusClassError
}
}
func (client *memdClient) resolveRequest(resp *memdQResponse) {
defer memd.ReleasePacket(resp.Packet)
logSchedf("Handling response data. OP=0x%x. Opaque=%d. Status:%d", resp.Command, resp.Opaque, resp.Status)
stClass := client.classifyResponseStatusClass(resp.Status)
client.lock.Lock()
// Find the request that goes with this response, don't check if the client is
// closed so that we can handle orphaned responses.
req := client.opList.FindAndMaybeRemove(resp.Opaque, stClass == statusClassError)
client.lock.Unlock()
if atomic.LoadUint32(&client.gracefulCloseTriggered) == 1 {
client.lock.Lock()
size := client.opList.Size()
client.lock.Unlock()
if size == 0 {
// Let's make sure that we don't somehow slow down returning to the user here.
go func() {
// We use the Close function rather than closeConn to ensure that we don't try to close the
// connection/client if someone else has already closed it.
err := client.Close()
if err != nil {
logDebugf("Failed to shutdown memdclient (%s) during graceful close: %s", client.loggerID(), err)
}
}()
}
}
if req == nil {
// There is no known request that goes with this response. Ignore it.
logDebugf("%s memdclient received response with no corresponding request.", client.loggerID())
if client.zombieLogger != nil {
client.zombieLogger.RecordZombieResponse(resp, client.connID, client.LocalAddress(), client.Address())
}
return
}
if !req.Persistent || stClass == statusClassError {
atomic.CompareAndSwapPointer(&req.waitingIn, unsafe.Pointer(client), nil)
}
req.processingLock.Lock()
req.AddResourceUnits(resp.ReadUnitsFrame, resp.WriteUnitsFrame)
if !req.Persistent {
stopNetTrace(req, resp, client.conn.LocalAddr(), client.conn.RemoteAddr())
}
isCompressed := (resp.Datatype & uint8(memd.DatatypeFlagCompressed)) != 0
if isCompressed && !client.disableDecompression {
newValue, err := snappy.Decode(nil, resp.Value)
if err != nil {
req.processingLock.Unlock()
logDebugf("%s memdclient failed to decompress value from the server for key `%s`.", client.loggerID(), req.Key)
return
}
resp.Value = newValue
resp.Datatype = resp.Datatype & ^uint8(memd.DatatypeFlagCompressed)
}
// Give the agent an opportunity to intercept the response first
var err error
if resp.Magic == memd.CmdMagicRes && stClass == statusClassError {
err = getKvStatusCodeError(resp.Status)
}
if client.breaker.CompletionCallback(err) {
client.breaker.MarkSuccessful()
} else {
client.breaker.MarkFailure()
}
if !req.Persistent {
stopCmdTrace(req)
}
req.processingLock.Unlock()
if err != nil {
shortCircuited, routeErr := client.postErrHandler(resp, req, err)
if shortCircuited {
logSchedf("Routing callback intercepted response")
return
}
err = routeErr
}
// Call the requests callback handler...
logSchedf("Dispatching response callback. OP=0x%x. Opaque=%d", resp.Command, resp.Opaque)
req.tryCallback(resp, err)
}
func (client *memdClient) run() {
var (
// A queue for DCP commands so we can execute them out-of-band from packet receiving. This
// is integral to allow the higher level application to back-pressure against the DCP packet
// processing without interfeering with the SDKs control commands (like config fetch).
dcpBufferQ = make(chan *dcpBuffer, client.dcpQueueSize)
// After we signal that DCP processing should stop, we need a notification so we know when
// it has been completed, we do this to prevent leaving the goroutine around, and we need to
// ensure that the application has finished with the last packet it received before we stop.
dcpProcDoneCh = make(chan struct{})
)
go func() {
defer close(dcpProcDoneCh)
for {
// If the client has been told to close then we need to finish ASAP, otherwise if the dcpBufferQ has been
// closed then we'll flush the queue first.
q, stillOpen := <-dcpBufferQ
if !stillOpen || atomic.LoadUint32(&client.shutdownDCP) != 0 {
return
}
logSchedf("Resolving response OP=0x%x. Opaque=%d", q.resp.Command, q.resp.Opaque)
client.resolveRequest(q.resp)
// See below for information on MB-26363 for why this is here.
if !q.isInternal && client.dcpAckSize > 0 {
client.maybeSendDcpBufferAck(q.packetLen)
}
}
}()
go func() {
for {
packet, n, err := client.conn.ReadPacket()
if err != nil {
client.lock.Lock()
if !client.closed {
logWarnf("%p memdClient read failure on conn `%v` : %v", client, client.connID, err)
}
client.lock.Unlock()
break
}
resp := &memdQResponse{
remoteAddr: client.conn.LocalAddr(),
sourceAddr: client.conn.RemoteAddr(),
sourceConnID: client.connID,
Packet: packet,
}
atomic.StoreInt64(&client.lastActivity, time.Now().UnixNano())
// We handle DCP no-op's directly here so we can reply immediately.
if resp.Packet.Command == memd.CmdDcpNoop {
err := client.conn.WritePacket(&memd.Packet{
Magic: memd.CmdMagicRes,
Command: memd.CmdDcpNoop,
Opaque: resp.Opaque,
})
if err != nil {
logWarnf("%p memdclient failed to dispatch DCP noop reply: %s", client, err)
}
continue
}
// This is a fix for a bug in the server DCP implementation (MB-26363). This
// bug causes the server to fail to send a stream-end notification. The server
// does however synchronously stop the stream, and thus we can assume no more
// packets will be received following the close response.
if resp.Magic == memd.CmdMagicRes && resp.Command == memd.CmdDcpCloseStream && client.streamEndNotSupported {
closeReq := client.opList.Find(resp.Opaque)
if closeReq != nil {
vbID := closeReq.Vbucket
streamReq := client.opList.FindOpenStream(vbID)
if streamReq != nil {
endExtras := make([]byte, 4)
binary.BigEndian.PutUint32(endExtras, uint32(memd.StreamEndClosed))
endResp := &memdQResponse{
Packet: &memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdDcpStreamEnd,
Vbucket: vbID,
Opaque: streamReq.Opaque,
Extras: endExtras,
},
}
dcpBufferQ <- &dcpBuffer{
resp: endResp,
packetLen: n,
isInternal: true,
}
}
}
}
switch resp.Packet.Command {
case memd.CmdDcpDeletion, memd.CmdDcpExpiration, memd.CmdDcpMutation, memd.CmdDcpSnapshotMarker,
memd.CmdDcpEvent, memd.CmdDcpOsoSnapshot, memd.CmdDcpSeqNoAdvanced, memd.CmdDcpStreamEnd:
dcpBufferQ <- &dcpBuffer{
resp: resp,
packetLen: n,
}
default:
logSchedf("%s memdclient resolving response OP=0x%x. Opaque=%d", client.loggerID(), resp.Command, resp.Opaque)
client.resolveRequest(resp)
}
}
client.lock.Lock()
if !client.closed {
client.closed = true
client.lock.Unlock()
err := client.closeConn(true)
if err != nil {
// Lets log a warning, as this is non-fatal
logWarnf("Failed to shut down client (%p) connection (%s)", client, err)
}
} else {
client.lock.Unlock()
}
// We close the buffer channel to wake the processor if its asleep (queue was empty).
// We then wait to ensure it is finished with whatever packet (or packets if the connection was closed by the
// server) was being processed.
close(dcpBufferQ)
<-dcpProcDoneCh
close(client.connReleaseNotify)
client.opList.Drain(func(req *memdQRequest) {
if !atomic.CompareAndSwapPointer(&req.waitingIn, unsafe.Pointer(client), nil) {
logWarnf("Encountered an unowned request in a client (%p) opMap", client)
}
shortCircuited, routeErr := client.postErrHandler(nil, req, io.EOF)
if shortCircuited {
return
}
req.tryCallback(nil, routeErr)
})
<-client.connReleasedNotify
close(client.closeNotify)
}()
}
func (client *memdClient) LocalAddress() string {
return client.conn.LocalAddr()
}
func (client *memdClient) GracefulClose(err error) {
if atomic.CompareAndSwapUint32(&client.gracefulCloseTriggered, 0, 1) {
client.lock.Lock()
if client.closed {
client.lock.Unlock()
return
}
persistentReqs := client.opList.FindAndRemoveAllPersistent()
client.lock.Unlock()
if err == nil {
err = io.EOF
}
for _, req := range persistentReqs {
req.cancelWithCallback(err)
}
// Close down the DCP worker, there can't be any future DCP messages. We don't
// strictly need to do this, as connection close will trigger it to close anyway.
atomic.StoreUint32(&client.shutdownDCP, 1)
client.lock.Lock()
size := client.opList.Size()
if size > 0 {
// If there are items in the op list then we need to go into graceful shutdown mode, so don't close anything
// yet.
client.lock.Unlock()
return
}
// If there are no items in the oplist then it's safe to close down the client and connection now.
if client.closed {
client.lock.Unlock()
return
}
client.closed = true
client.lock.Unlock()
err := client.closeConn(false)
if err != nil {
// Lets log a warning, as this is non-fatal
logWarnf("Failed to shut down client (%p) connection (%s)", client, err)
}
}
}
func (client *memdClient) closeConn(internalTrigger bool) error {
logDebugf("%s memdclient closing connection, internal close: %t", client.loggerID(), internalTrigger)
err := client.conn.Close()
if err != nil {
logDebugf("Failed to close memdconn: %v on memdclient %s", err, client.loggerID())
}
// If this has been triggered by the read side failing a read before the client is closed then we
// can be certain that we aren't going to attempt a read, and it's safe to release the connection.
// Otherwise, we need to wait for the connection close to propagate through the read side and to be told
// that reading has stopped so we can safely release.
if !internalTrigger {
<-client.connReleaseNotify
}
client.conn.Release()
close(client.connReleasedNotify)
return err
}
func (client *memdClient) Close() error {
// We mark that we are shutting down to stop the DCP processor from running any
// additional packets up to the application. We do this before the closed check to
// force stop flushing. Rebalance etc... uses GracefulClose so if we received this Close
// then we do need to shutdown in a timely manner.
atomic.StoreUint32(&client.shutdownDCP, 1)
client.lock.Lock()
if client.closed {
client.lock.Unlock()
return nil
}
client.closed = true
client.lock.Unlock()
return client.closeConn(false)
}
func (client *memdClient) sendCanary() {
errChan := make(chan error)
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
errChan <- err
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdNoop,
Datatype: 0,
Cas: 0,
Key: nil,
Value: nil,
},
Callback: handler,
RetryStrategy: newFailFastRetryStrategy(),
}
logDebugf("Sending NOOP request for %s", client.loggerID())
err := client.internalSendRequest(req)
if err != nil {
client.breaker.MarkFailure()
}
timer := AcquireTimer(client.breaker.CanaryTimeout())
select {
case <-timer.C:
if !req.internalCancel(errRequestCanceled) {
err := <-errChan
if err == nil {
logDebugf("NOOP request successful for %s", client.loggerID())
client.breaker.MarkSuccessful()
} else {
logDebugf("NOOP request failed for %s", client.loggerID())
client.breaker.MarkFailure()
}
}
client.breaker.MarkFailure()
case err := <-errChan:
if err == nil {
client.breaker.MarkSuccessful()
} else {
client.breaker.MarkFailure()
}
}
}
func (client *memdClient) loggerID() string {
return fmt.Sprintf("%s/%p", client.Address(), client)
}
gocbcore-10.2.3/memdclientdialer_component.go 0000664 0000000 0000000 00000055077 14417540156 0021320 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"context"
"crypto/tls"
"errors"
"sync"
"sync/atomic"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
type helloProps struct {
MutationTokensEnabled bool
CollectionsEnabled bool
CompressionEnabled bool
DurationsEnabled bool
OutOfOrderEnabled bool
JSONFeatureEnabled bool
XErrorFeatureEnabled bool
SyncReplicationEnabled bool
PITRFeatureEnabled bool
ResourceUnitsEnabled bool
}
type bootstrapProps struct {
Bucket string
UserAgent string
ErrMapManager *errMapComponent
HelloProps helloProps
}
type memdClientDialerComponent struct {
kvConnectTimeout time.Duration
serverWaitTimeout time.Duration
clientID string
breakerCfg CircuitBreakerConfig
compressionMinSize int
compressionMinRatio float64
disableDecompression bool
connBufSize uint
serverFailuresLock sync.Mutex
serverFailures map[string]time.Time
tracer *tracerComponent
zombieLogger *zombieLoggerComponent
bootstrapProps bootstrapProps
bootstrapFailHandlersLock sync.Mutex
bootstrapFailHandlers []memdBoostrapFailHandler
cccpUnsupportedHandlersLock sync.Mutex
cccpUnsupportedFailHandlers []memdBoostrapCCCPUnsupportedHandler
configApplied uint32
noTLSSeedNode bool
dcpBootstrapProps *memdBootstrapDCPProps
dcpQueueSize int
cfgManager *configManagementComponent
}
type memdBootstrapDCPProps struct {
disableBufferAcknowledgement bool
useOSOBackfill bool
useStreamID bool
useExpiryOpcode bool
backfillOrderStr string
priorityStr string
streamName string
openFlags memd.DcpOpenFlag
bufferSize int
}
type memdClientDialerProps struct {
KVConnectTimeout time.Duration
ServerWaitTimeout time.Duration
ClientID string
CompressionMinSize int
CompressionMinRatio float64
DisableDecompression bool
NoTLSSeedNode bool
ConnBufSize uint
DCPBootstrapProps *memdBootstrapDCPProps
DCPQueueSize int
}
type memdBoostrapFailHandler interface {
onBootstrapFail(error)
}
type memdBoostrapCCCPUnsupportedHandler interface {
onCCCPUnsupported(error)
}
func newMemdClientDialerComponent(props memdClientDialerProps, bSettings bootstrapProps, breakerCfg CircuitBreakerConfig,
zLogger *zombieLoggerComponent, tracer *tracerComponent, cfgManager *configManagementComponent) *memdClientDialerComponent {
dialer := &memdClientDialerComponent{
kvConnectTimeout: props.KVConnectTimeout,
serverWaitTimeout: props.ServerWaitTimeout,
clientID: props.ClientID,
breakerCfg: breakerCfg,
zombieLogger: zLogger,
tracer: tracer,
serverFailures: make(map[string]time.Time),
bootstrapProps: bSettings,
dcpBootstrapProps: props.DCPBootstrapProps,
dcpQueueSize: props.DCPQueueSize,
compressionMinSize: props.CompressionMinSize,
compressionMinRatio: props.CompressionMinRatio,
disableDecompression: props.DisableDecompression,
noTLSSeedNode: props.NoTLSSeedNode,
connBufSize: props.ConnBufSize,
cfgManager: cfgManager,
}
cfgManager.AddConfigWatcher(dialer)
return dialer
}
func (mcc *memdClientDialerComponent) ResetConfig() {
atomic.StoreUint32(&mcc.configApplied, 0)
mcc.cfgManager.AddConfigWatcher(mcc)
}
func (mcc *memdClientDialerComponent) OnNewRouteConfig(cfg *routeConfig) {
if cfg.revID == -1 {
return
}
atomic.StoreUint32(&mcc.configApplied, 1)
mcc.cfgManager.RemoveConfigWatcher(mcc)
}
func (mcc *memdClientDialerComponent) AddBootstrapFailHandler(handler memdBoostrapFailHandler) {
mcc.bootstrapFailHandlersLock.Lock()
mcc.bootstrapFailHandlers = append(mcc.bootstrapFailHandlers, handler)
mcc.bootstrapFailHandlersLock.Unlock()
}
func (mcc *memdClientDialerComponent) AddCCCPUnsupportedHandler(handler memdBoostrapCCCPUnsupportedHandler) {
mcc.cccpUnsupportedHandlersLock.Lock()
mcc.cccpUnsupportedFailHandlers = append(mcc.cccpUnsupportedFailHandlers, handler)
mcc.cccpUnsupportedHandlersLock.Unlock()
}
func (mcc *memdClientDialerComponent) RemoveBootstrapFailHandler(handler memdBoostrapFailHandler) {
var idx int
mcc.bootstrapFailHandlersLock.Lock()
for i, w := range mcc.bootstrapFailHandlers {
if w == handler {
idx = i
}
}
if idx == len(mcc.bootstrapFailHandlers) {
mcc.bootstrapFailHandlers = mcc.bootstrapFailHandlers[:idx]
} else {
mcc.bootstrapFailHandlers = append(mcc.bootstrapFailHandlers[:idx], mcc.bootstrapFailHandlers[idx+1:]...)
}
mcc.bootstrapFailHandlersLock.Unlock()
}
func (mcc *memdClientDialerComponent) SlowDialMemdClient(cancelSig <-chan struct{}, address routeEndpoint, tlsConfig *dynTLSConfig,
auth AuthProvider, authMechanisms []AuthMechanism, postCompleteHandler postCompleteErrorHandler) (*memdClient, error) {
mcc.serverFailuresLock.Lock()
failureTime := mcc.serverFailures[address.Address]
mcc.serverFailuresLock.Unlock()
if !failureTime.IsZero() {
waitedTime := time.Since(failureTime)
if waitedTime < mcc.serverWaitTimeout {
select {
case <-cancelSig:
return nil, errRequestCanceled
case <-time.After(mcc.serverWaitTimeout - waitedTime):
}
}
}
deadline := time.Now().Add(mcc.kvConnectTimeout)
client, err := mcc.dialMemdClient(cancelSig, address, deadline, postCompleteHandler, tlsConfig)
if err != nil {
if !errors.Is(err, ErrRequestCanceled) {
mcc.serverFailuresLock.Lock()
mcc.serverFailures[address.Address] = time.Now()
mcc.serverFailuresLock.Unlock()
}
return nil, err
}
bClient := newMemdBootstrapClient(client, cancelSig)
if mcc.dcpBootstrapProps == nil {
err = mcc.bootstrap(bClient, deadline, authMechanisms, auth)
} else {
err = mcc.dcpBootstrap(newDCPBootstrapClient(bClient), deadline, authMechanisms, auth)
}
if err != nil {
closeErr := client.Close()
if closeErr != nil {
logWarnf("Failed to close authentication client (%s)", closeErr)
}
if !errors.Is(err, ErrForcedReconnect) {
mcc.serverFailuresLock.Lock()
mcc.serverFailures[address.Address] = time.Now()
mcc.serverFailuresLock.Unlock()
}
mcc.bootstrapFailHandlersLock.Lock()
handlers := make([]memdBoostrapFailHandler, len(mcc.bootstrapFailHandlers))
copy(handlers, mcc.bootstrapFailHandlers)
mcc.bootstrapFailHandlersLock.Unlock()
for _, handler := range handlers {
handler.onBootstrapFail(err)
}
return nil, err
}
return client, nil
}
func (mcc *memdClientDialerComponent) dialMemdClient(cancelSig <-chan struct{}, address routeEndpoint, deadline time.Time,
postCompleteHandler postCompleteErrorHandler, dynTls *dynTLSConfig) (*memdClient, error) {
// Copy the tls configuration since we need to provide the hostname for each
// server that we connect to so that the certificate can be validated properly.
var tlsConfig *tls.Config
if dynTls != nil && !(mcc.noTLSSeedNode && address.IsSeedNode) {
srvTLSConfig, err := dynTls.MakeForAddr(address.Address)
if err != nil {
return nil, err
}
tlsConfig = srvTLSConfig
}
ctx, cancel := context.WithCancel(context.Background())
go func() {
select {
case <-ctx.Done():
return
case <-cancelSig:
cancel()
}
}()
conn, err := dialMemdConn(ctx, address.Address, tlsConfig, deadline, mcc.connBufSize)
cancel()
if err != nil {
if errors.Is(err, context.Canceled) {
err = errRequestCanceled
} else {
err = wrapError(err, "check server ports and cluster encryption setting")
}
logDebugf("Failed to connect. %v", err)
return nil, err
}
client := newMemdClient(
memdClientProps{
ClientID: mcc.clientID,
DCPQueueSize: mcc.dcpQueueSize,
DisableDecompression: mcc.disableDecompression,
CompressionMinRatio: mcc.compressionMinRatio,
CompressionMinSize: mcc.compressionMinSize,
},
conn,
mcc.breakerCfg,
postCompleteHandler,
mcc.tracer,
mcc.zombieLogger,
)
return client, err
}
func (mcc *memdClientDialerComponent) dcpBootstrap(client *dcpBootstrapClient, deadline time.Time,
authMechanisms []AuthMechanism, authProvider AuthProvider) error {
if err := mcc.bootstrap(client, deadline, authMechanisms, authProvider); err != nil {
return err
}
if err := client.ExecOpenDcpConsumer(mcc.dcpBootstrapProps.streamName, mcc.dcpBootstrapProps.openFlags, deadline); err != nil {
return err
}
if err := client.ExecEnableDcpNoop(180*time.Second, deadline); err != nil {
return err
}
if mcc.dcpBootstrapProps.priorityStr != "" {
if err := client.ExecDcpControl("set_priority", mcc.dcpBootstrapProps.priorityStr, deadline); err != nil {
return err
}
}
if mcc.dcpBootstrapProps.useExpiryOpcode {
if err := client.ExecDcpControl("enable_expiry_opcode", "true", deadline); err != nil {
return err
}
}
if mcc.dcpBootstrapProps.useStreamID {
if err := client.ExecDcpControl("enable_stream_id", "true", deadline); err != nil {
return err
}
}
if mcc.dcpBootstrapProps.useOSOBackfill {
if err := client.ExecDcpControl("enable_out_of_order_snapshots", "true", deadline); err != nil {
return err
}
}
if mcc.dcpBootstrapProps.backfillOrderStr != "" {
if err := client.ExecDcpControl("backfill_order", mcc.dcpBootstrapProps.backfillOrderStr, deadline); err != nil {
return err
}
}
if !mcc.dcpBootstrapProps.disableBufferAcknowledgement {
if err := client.ExecEnableDcpBufferAck(mcc.dcpBootstrapProps.bufferSize, deadline); err != nil {
return err
}
}
return client.ExecEnableDcpClientEnd(deadline)
}
func (mcc *memdClientDialerComponent) bootstrap(client bootstrapClient, deadline time.Time,
authMechanisms []AuthMechanism, authProvider AuthProvider) error {
logDebugf("Memdclient `%s/%p` Fetching cluster client data", client.Address(), client)
bucket := mcc.bootstrapProps.Bucket
features := helloFeatures(mcc.bootstrapProps.HelloProps)
clientInfoStr := clientInfoString(client.ConnID(), mcc.bootstrapProps.UserAgent)
helloCh, err := client.ExecHello(clientInfoStr, features, deadline)
if err != nil {
logDebugf("Memdclient `%s/%p` Failed to execute HELLO (%v)", client.Address(), client, err)
return err
}
errMapCh, err := client.ExecGetErrorMap(2, deadline)
if err != nil {
// GetErrorMap isn't integral to bootstrap succeeding
logDebugf("Memdclient `%s/%p`Failed to execute Get error map (%v)", client.Address(), client, err)
}
var listMechsCh chan SaslListMechsCompleted
var completedAuthCh chan error
var continueAuthCh chan bool
firstAuthMethod := mcc.buildAuthHandler(client, authProvider, deadline, authMechanisms[0])
if firstAuthMethod != nil {
// If the auth method is nil then we don't actually need to do any auth so no need to Get the mechanisms.
listMechsCh = make(chan SaslListMechsCompleted, 1)
err = client.SaslListMechs(deadline, func(mechs []AuthMechanism, err error) {
if err != nil {
logDebugf("Memdclient `%s/%p` Failed to fetch list auth mechs (%v)", client.Address(), client, err)
}
listMechsCh <- SaslListMechsCompleted{
Err: err,
Mechs: mechs,
}
})
if err != nil {
logDebugf("Memdclient `%s/%p` Failed to execute list auth mechs (%v)", client.Address(), client, err)
}
completedAuthCh, continueAuthCh, err = firstAuthMethod()
if err != nil {
logDebugf("Memdclient `%s/%p` Failed to execute auth (%v)", client.Address(), client, err)
return err
}
}
var selectCh chan error
var configCh chan getConfigResponse
// If there's no bucket then we don't need to do select bucket, we also don't need to wait for the continue channel,
// as it will never be read and will be garbage collected.
if continueAuthCh == nil {
if bucket != "" {
selectCh, err = client.ExecSelectBucket([]byte(bucket), deadline)
if err != nil {
logDebugf("Memdclient `%s/%p` Failed to execute select bucket (%v)", client.Address(), client, err)
return err
}
}
if atomic.LoadUint32(&mcc.configApplied) == 0 {
configCh, err = client.ExecGetConfig(deadline)
if err != nil {
// Getting a config isn't essential to bootstrap.
logDebugf("Memdclient `%s/%p` Failed to execute get config (%v)", client.Address(), client, err)
}
}
} else {
selectCh, configCh = mcc.continueAfterAuth(client, bucket, continueAuthCh, deadline)
}
helloResp := <-helloCh
if helloResp.Err != nil {
logDebugf("Memdclient `%s/%p` Failed to hello with server (%v)", client.Address(), client, helloResp.Err)
return helloResp.Err
}
if errMapCh != nil {
errMapResp := <-errMapCh
if errMapResp.Err == nil {
mcc.bootstrapProps.ErrMapManager.StoreErrorMap(errMapResp.Bytes)
} else {
logDebugf("Memdclient `%s/%p` Failed to fetch kv error map (%s)", client.Address(), client, errMapResp.Err)
}
}
var serverAuthMechanisms []AuthMechanism
if listMechsCh != nil {
listMechsResp := <-listMechsCh
if listMechsResp.Err == nil {
serverAuthMechanisms = listMechsResp.Mechs
logDebugf("Memdclient `%s/%p` Server supported auth mechanisms: %v", client.Address(), client, serverAuthMechanisms)
} else {
logDebugf("Memdclient `%s/%p` Failed to fetch auth mechs from server (%v)", client.Address(), client, listMechsResp.Err)
}
}
// If completedAuthCh isn't nil then we have attempted to do auth so we need to wait on the result of that.
if completedAuthCh != nil {
authErr := <-completedAuthCh
if authErr != nil {
logDebugf("Memdclient `%s/%p` Failed to perform auth against server (%v)", client.Address(), client, authErr)
if errors.Is(authErr, ErrRequestCanceled) {
// There's no point in us trying different mechanisms if something has cancelled bootstrapping.
return authErr
} else if errors.Is(authErr, ErrAuthenticationFailure) {
// If there's only one auth mechanism then we can just fail.
if len(authMechanisms) == 1 {
return authErr
}
// If the server supports the mechanism we've tried then this auth error can't be due to an unsupported
// mechanism.
for _, mech := range serverAuthMechanisms {
if mech == authMechanisms[0] {
return authErr
}
}
// If we've got here then the auth mechanism we tried is unsupported so let's keep trying with the next
// supported mechanism.
logInfof("Memdclient `%p` Unsupported authentication mechanism, will attempt to find next supported mechanism", client)
}
for {
var found bool
var mech AuthMechanism
found, mech, authMechanisms = findNextAuthMechanism(authMechanisms, serverAuthMechanisms)
if !found {
logDebugf("Memdclient `%s/%p` Failed to authenticate, all options exhausted", client.Address(), client)
return authErr
}
logDebugf("Memdclient `%s/%p` Retrying authentication with found supported mechanism: %s", client.Address(), client, mech)
nextAuthFunc := mcc.buildAuthHandler(client, authProvider, deadline, mech)
if nextAuthFunc == nil {
// This can't really happen but just in case it somehow does.
logInfof("Memdclient `%p` Failed to authenticate, no available credentials", client)
return authErr
}
completedAuthCh, continueAuthCh, err = nextAuthFunc()
if err != nil {
logDebugf("Memdclient `%s/%p` Failed to execute auth (%v)", client.Address(), client, err)
return err
}
if continueAuthCh == nil {
if bucket != "" {
selectCh, err = client.ExecSelectBucket([]byte(bucket), deadline)
if err != nil {
logDebugf("Memdclient `%s/%p` Failed to execute select bucket (%v)", client.Address(), client, err)
return err
}
}
if atomic.LoadUint32(&mcc.configApplied) == 0 {
configCh, err = client.ExecGetConfig(deadline)
if err != nil {
// Getting a config isn't essential to bootstrap.
logDebugf("Memdclient `%s/%p` Failed to execute get config (%v)", client.Address(), client, err)
}
}
} else {
selectCh, configCh = mcc.continueAfterAuth(client, bucket, continueAuthCh, deadline)
}
authErr = <-completedAuthCh
if authErr == nil {
break
}
logDebugf("Memdclient `%s/%p` Failed to perform auth against server (%v)", client.Address(), client, authErr)
if errors.Is(authErr, ErrAuthenticationFailure) || errors.Is(err, ErrRequestCanceled) {
return authErr
}
}
}
logDebugf("Memdclient `%s/%p` Authenticated successfully", client.Address(), client)
}
var selectErr error
if selectCh != nil {
selectErr = <-selectCh
}
// If we've done a config fetch then we try to read the result of that before checking if select bucket succeeded.
// We might have managed to get a config even if select bucket failed, e.g. if we're bootstrapping against a non-kv
// node.
if configCh != nil {
configResp := <-configCh
err = configResp.Err
if err == nil {
// We don't want this to block us completing bootstrap.
go mcc.cfgManager.OnNewConfig(configResp.Config)
} else {
logDebugf("Memdclient `%s/%p` Failed to perform config fetch against server (%v)", client.Address(), client, err)
if errors.Is(err, ErrDocumentNotFound) {
logDebugf("Memdclient `%s/%p` detected that CCCP is unsupported, informing upstream", client.Address(), client)
mcc.sendErrorToCCCPUnsupportedHandlers()
}
}
}
if selectErr != nil {
logDebugf("Memdclient `%s/%p` Failed to perform select bucket against server (%v)", client.Address(), client, selectErr)
return selectErr
}
client.Features(helloResp.SrvFeatures)
logDebugf("Memdclient `%s/%p` Client Features: %+v", client.Address(), client, features)
logDebugf("Memdclient `%s/%p` Server Features: %+v", client.Address(), client, helloResp.SrvFeatures)
return nil
}
func (mcc *memdClientDialerComponent) continueAfterAuth(client bootstrapClient, bucketName string, continueAuthCh chan bool,
deadline time.Time) (chan error, chan getConfigResponse) {
var selectCh chan error
if bucketName != "" {
selectCh = make(chan error, 1)
}
var configCh chan getConfigResponse
if atomic.LoadUint32(&mcc.configApplied) == 0 {
configCh = make(chan getConfigResponse, 1)
}
go func() {
success := <-continueAuthCh
if !success {
if selectCh != nil {
close(selectCh)
}
if configCh != nil {
close(configCh)
}
return
}
var execCh chan error
if selectCh != nil {
var err error
execCh, err = client.ExecSelectBucket([]byte(bucketName), deadline)
if err != nil {
logDebugf("Memdclient `%s/%p` Failed to execute select bucket (%v)", client.Address(), client, err)
selectCh <- err
return
}
}
var execConfigCh chan getConfigResponse
if configCh != nil {
var err error
execConfigCh, err = client.ExecGetConfig(deadline)
if err != nil {
// Getting a config isn't essential to bootstrap.
logDebugf("Memdclient `%s/%p` Failed to execute get config (%v)", client.Address(), client, err)
close(configCh)
return
}
}
if selectCh != nil {
execErr := <-execCh
selectCh <- execErr
}
if configCh != nil {
configResp := <-execConfigCh
configCh <- configResp
}
}()
return selectCh, configCh
}
type authFunc func() (continueCh chan error, completedCb chan bool, err error)
func (mcc *memdClientDialerComponent) buildAuthHandler(client bootstrapClient, auth AuthProvider, deadline time.Time,
mechanism AuthMechanism) authFunc {
creds, err := getKvAuthCreds(auth, client.Address())
if err != nil {
return nil
}
if creds.Username != "" || creds.Password != "" {
return func() (chan error, chan bool, error) {
continueCh := make(chan bool, 1)
completedCh := make(chan error, 1)
hasContinued := int32(0)
callErr := saslMethod(mechanism, creds.Username, creds.Password, client, deadline, func() {
// hasContinued should never be 1 here but let's guard against it.
if atomic.CompareAndSwapInt32(&hasContinued, 0, 1) {
continueCh <- true
}
}, func(err error) {
if atomic.CompareAndSwapInt32(&hasContinued, 0, 1) {
sendContinue := true
if err != nil {
sendContinue = false
}
continueCh <- sendContinue
}
completedCh <- err
})
if callErr != nil {
return nil, nil, callErr
}
return completedCh, continueCh, nil
}
}
return nil
}
func (mcc *memdClientDialerComponent) sendErrorToCCCPUnsupportedHandlers() {
mcc.cccpUnsupportedHandlersLock.Lock()
handlers := make([]memdBoostrapCCCPUnsupportedHandler, len(mcc.cccpUnsupportedFailHandlers))
copy(handlers, mcc.cccpUnsupportedFailHandlers)
mcc.cccpUnsupportedHandlersLock.Unlock()
for _, h := range handlers {
h.onCCCPUnsupported(ErrUnsupportedOperation)
}
}
func checkSupportsFeature(srvFeatures []memd.HelloFeature, feature memd.HelloFeature) bool {
for _, srvFeature := range srvFeatures {
if srvFeature == feature {
return true
}
}
return false
}
func findNextAuthMechanism(authMechanisms []AuthMechanism, serverAuthMechanisms []AuthMechanism) (bool, AuthMechanism, []AuthMechanism) {
for {
if len(authMechanisms) <= 1 {
break
}
authMechanisms = authMechanisms[1:]
mech := authMechanisms[0]
for _, serverMech := range serverAuthMechanisms {
if mech == serverMech {
return true, mech, authMechanisms
}
}
}
return false, "", authMechanisms
}
func helloFeatures(props helloProps) []memd.HelloFeature {
var features []memd.HelloFeature
// Send the TLS flag, which has unknown effects.
features = append(features, memd.FeatureTLS)
// Indicate that we understand XATTRs
features = append(features, memd.FeatureXattr)
// Indicates that we understand select buckets.
features = append(features, memd.FeatureSelectBucket)
// If the user wants to use KV Error maps, lets enable them
if props.XErrorFeatureEnabled {
features = append(features, memd.FeatureXerror)
}
// Indicate that we understand JSON
if props.JSONFeatureEnabled {
features = append(features, memd.FeatureJSON)
}
// Indicate that we understand Point in Time
if props.PITRFeatureEnabled {
features = append(features, memd.FeaturePITR)
}
// If the user wants to use mutation tokens, lets enable them
if props.MutationTokensEnabled {
features = append(features, memd.FeatureSeqNo)
}
// If the user wants on-the-wire compression, lets try to enable it
if props.CompressionEnabled {
features = append(features, memd.FeatureSnappy)
}
if props.DurationsEnabled {
features = append(features, memd.FeatureDurations)
}
if props.CollectionsEnabled {
features = append(features, memd.FeatureCollections)
}
if props.OutOfOrderEnabled {
features = append(features, memd.FeatureUnorderedExec)
}
// These flags are informational so don't actually enable anything
features = append(features, memd.FeatureAltRequests)
features = append(features, memd.FeatureCreateAsDeleted)
features = append(features, memd.FeatureReplaceBodyWithXattr)
features = append(features, memd.FeaturePreserveExpiry)
if props.SyncReplicationEnabled {
features = append(features, memd.FeatureSyncReplication)
}
if props.ResourceUnitsEnabled {
features = append(features, memd.FeatureResourceUnits)
}
return features
}
gocbcore-10.2.3/memdconn.go 0000664 0000000 0000000 00000007634 14417540156 0015530 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"bufio"
"context"
"crypto/tls"
"io"
"net"
"sync"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
const defaultReaderBufSize = 20 * 1024 * 1024
type memdConn interface {
LocalAddr() string
RemoteAddr() string
WritePacket(*memd.Packet) error
ReadPacket() (*memd.Packet, int, error)
Close() error
Release()
EnableFeature(feature memd.HelloFeature)
IsFeatureEnabled(feature memd.HelloFeature) bool
}
type wrappedReadWriteCloser struct {
*bufio.Reader
io.Writer
io.Closer
}
// readerBufPools - Map of buffer size to thread safe pool containing packet reader buffers.
var readerBufPools = map[int]*sync.Pool{}
var readerBufPoolsLock sync.Mutex
// acquireReadBuf - Returns a pointer to a read buffer which is ready to be used, ensure the buffer is released using
// the 'releaseWriteBuf' function.
func acquireReadBuf(stream io.Reader, bufSize int) *bufio.Reader {
readerBufPoolsLock.Lock()
bufPool, ok := readerBufPools[bufSize]
if !ok {
bufPool = &sync.Pool{}
readerBufPools[bufSize] = bufPool
}
readerBufPoolsLock.Unlock()
iReader := bufPool.Get()
var reader *bufio.Reader
if iReader == nil {
reader = bufio.NewReaderSize(stream, bufSize)
} else {
var ok bool
reader, ok = iReader.(*bufio.Reader)
if ok {
reader.Reset(stream)
} else {
reader = bufio.NewReaderSize(stream, bufSize)
}
}
return reader
}
// releaseReadBuf - Reset the buffer so that it's clean for the next user (note that this retains the underlying
// storage for future reads) and then return it to the pool.
func releaseReadBuf(buf *bufio.Reader, bufSize int) {
buf.Reset(nil)
readerBufPoolsLock.Lock()
bufPool, ok := readerBufPools[bufSize]
if !ok {
readerBufPoolsLock.Unlock()
logWarnf("Attempted to release a read buffer for a buffer size without a registered pool")
return
}
bufPool.Put(buf)
readerBufPoolsLock.Unlock()
}
type memdConnWrap struct {
localAddr string
remoteAddr string
conn *memd.Conn
baseConn *wrappedReadWriteCloser
bufSize int
}
func (s *memdConnWrap) LocalAddr() string {
return s.localAddr
}
func (s *memdConnWrap) RemoteAddr() string {
return s.remoteAddr
}
func (s *memdConnWrap) WritePacket(pkt *memd.Packet) error {
return s.conn.WritePacket(pkt)
}
func (s *memdConnWrap) ReadPacket() (*memd.Packet, int, error) {
return s.conn.ReadPacket()
}
func (s *memdConnWrap) EnableFeature(feature memd.HelloFeature) {
s.conn.EnableFeature(feature)
}
func (s *memdConnWrap) IsFeatureEnabled(feature memd.HelloFeature) bool {
return s.conn.IsFeatureEnabled(feature)
}
func (s *memdConnWrap) Close() error {
return s.baseConn.Close()
}
// Release is not thread safe and should not be called whilst there are pending calls, such as ReadPacket.
func (s *memdConnWrap) Release() {
if s.baseConn == nil {
logWarnf("Release called on already released connection")
return
}
releaseReadBuf(s.baseConn.Reader, s.bufSize)
s.baseConn = nil
}
func dialMemdConn(ctx context.Context, address string, tlsConfig *tls.Config, deadline time.Time, bufSize uint) (memdConn, error) {
d := net.Dialer{
Deadline: deadline,
}
baseConn, err := d.DialContext(ctx, "tcp", address)
if err != nil {
return nil, err
}
tcpConn, isTCPConn := baseConn.(*net.TCPConn)
if !isTCPConn || tcpConn == nil {
return nil, errCliInternalError
}
err = tcpConn.SetNoDelay(false)
if err != nil {
logWarnf("Failed to disable TCP nodelay (%s)", err)
}
var conn io.ReadWriteCloser = tcpConn
if tlsConfig != nil {
tlsConn := tls.Client(tcpConn, tlsConfig)
err = tlsConn.Handshake()
if err != nil {
return nil, err
}
conn = tlsConn
}
if bufSize == 0 {
bufSize = defaultReaderBufSize
}
c := &wrappedReadWriteCloser{
Reader: acquireReadBuf(conn, int(bufSize)),
Writer: conn,
Closer: conn,
}
return &memdConnWrap{
conn: memd.NewConn(c),
baseConn: c,
localAddr: baseConn.LocalAddr().String(),
remoteAddr: address,
bufSize: int(bufSize),
}, nil
}
gocbcore-10.2.3/memdopmap.go 0000664 0000000 0000000 00000005104 14417540156 0015675 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"sync/atomic"
"github.com/couchbase/gocbcore/v10/memd"
)
// memdOpMap - Uses the requests opaque to map requests to responses. Note that this structure is not thread safe, and
// uses should be guarded by a mutex.
type memdOpMap struct {
opaque uint32
requests map[uint32]*memdQRequest
}
// newMemdOpMap - Creates a new empty 'memdOpMap' initializing any internal structures. Note that the requests opaque
// will begin at one and monotonically increase from there.
func newMemdOpMap() *memdOpMap {
return &memdOpMap{requests: make(map[uint32]*memdQRequest)}
}
// Add - Add a new request to the map, the provided requests opaque value will be updated atomically.
func (m *memdOpMap) Add(req *memdQRequest) {
m.opaque++
atomic.StoreUint32(&req.Opaque, m.opaque)
m.requests[m.opaque] = req
}
// Remove - Remove the provided request from the map.
func (m *memdOpMap) Remove(req *memdQRequest) bool {
_, ok := m.requests[req.Opaque]
delete(m.requests, req.Opaque)
return ok
}
// FindOpenStream - This allows searching through the list of requests for a specific request. This is only used to fix
// the DCP server bug MB-26363.
func (m *memdOpMap) FindOpenStream(vbID uint16) *memdQRequest {
for _, req := range m.requests {
if req.Magic == memd.CmdMagicReq && req.Command == memd.CmdDcpStreamReq && req.Vbucket == vbID {
return req
}
}
return nil
}
// FindAndRemoveAllPersistent - Find all persistent requests, removing them from the map and returning them all.
func (m *memdOpMap) FindAndRemoveAllPersistent() []*memdQRequest {
var reqs []*memdQRequest
for _, req := range m.requests {
if req.Persistent {
reqs = append(reqs, req)
delete(m.requests, req.Opaque)
}
}
return reqs
}
// Find - Lookup a request using its opaque, note that this function by return a pointer.
func (m *memdOpMap) Find(opaque uint32) *memdQRequest {
return m.requests[opaque]
}
// FindAndMaybeRemove - Lookup a request using its opaque and then remove it from the map if it's not persistent or the
// 'force' argument is true.
func (m *memdOpMap) FindAndMaybeRemove(opaque uint32, force bool) *memdQRequest {
req, ok := m.requests[opaque]
if !ok {
return nil
}
if force || !req.Persistent {
delete(m.requests, opaque)
}
return req
}
func (m *memdOpMap) Size() int {
return len(m.requests)
}
// Drain - Remove all the requests from the map whilst running the provided callback for each request.
func (m *memdOpMap) Drain(callback func(req *memdQRequest)) {
for _, req := range m.requests {
callback(req)
}
m.requests = make(map[uint32]*memdQRequest)
}
gocbcore-10.2.3/memdopmap_test.go 0000664 0000000 0000000 00000006214 14417540156 0016737 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"github.com/couchbase/gocbcore/v10/memd"
)
func (suite *StandardTestSuite) TestOpMap() {
rd := newMemdOpMap()
testOp1 := &memdQRequest{
Packet: memd.Packet{},
}
testOp2 := &memdQRequest{
Packet: memd.Packet{},
}
testOp3 := &memdQRequest{
Packet: memd.Packet{},
Persistent: true,
}
// Single Remove
rd.Add(testOp1)
if rd.Remove(testOp1) != true {
suite.T().Fatalf("The op should be there")
}
if rd.Remove(testOp1) != false {
suite.T().Fatalf("There should be nothing to remove")
}
// Single opaque remove
rd.Add(testOp1)
if rd.FindAndMaybeRemove(testOp1.Opaque, false) != testOp1 {
suite.T().Fatalf("The op should have been found")
}
if rd.FindAndMaybeRemove(testOp1.Opaque, false) != nil {
suite.T().Fatalf("The op should not have been there")
}
// In order remove
rd.Add(testOp1)
rd.Add(testOp2)
if rd.Remove(testOp1) != true {
suite.T().Fatalf("The op should be there")
}
if rd.Remove(testOp2) != true {
suite.T().Fatalf("The op should be there")
}
if rd.Remove(testOp1) != false {
suite.T().Fatalf("There should be nothing to remove")
}
if rd.Remove(testOp2) != false {
suite.T().Fatalf("There should be nothing to remove")
}
// Out of order remove
rd.Add(testOp1)
rd.Add(testOp2)
if rd.Remove(testOp2) != true {
suite.T().Fatalf("The op should be there")
}
if rd.Remove(testOp1) != true {
suite.T().Fatalf("The op should be there")
}
if rd.Remove(testOp2) != false {
suite.T().Fatalf("There should be nothing to remove")
}
if rd.Remove(testOp1) != false {
suite.T().Fatalf("There should be nothing to remove")
}
// In order opaque remove
rd.Add(testOp1)
rd.Add(testOp2)
if rd.FindAndMaybeRemove(testOp1.Opaque, false) != testOp1 {
suite.T().Fatalf("The op should have been found")
}
if rd.FindAndMaybeRemove(testOp2.Opaque, false) != testOp2 {
suite.T().Fatalf("The op should have been found")
}
if rd.FindAndMaybeRemove(testOp1.Opaque, false) != nil {
suite.T().Fatalf("The op should not have been there")
}
if rd.FindAndMaybeRemove(testOp2.Opaque, false) != nil {
suite.T().Fatalf("The op should not have been there")
}
// Out of order opaque remove
rd.Add(testOp1)
rd.Add(testOp2)
if rd.FindAndMaybeRemove(testOp2.Opaque, false) != testOp2 {
suite.T().Fatalf("The op should have been found")
}
if rd.FindAndMaybeRemove(testOp1.Opaque, false) != testOp1 {
suite.T().Fatalf("The op should have been found")
}
if rd.FindAndMaybeRemove(testOp2.Opaque, false) != nil {
suite.T().Fatalf("The op should not have been there")
}
if rd.FindAndMaybeRemove(testOp1.Opaque, false) != nil {
suite.T().Fatalf("The op should not have been there")
}
rd.Add(testOp3)
if rd.FindAndMaybeRemove(testOp3.Opaque, true) != testOp3 {
suite.T().Fatalf("The op should have been found")
}
if rd.FindAndMaybeRemove(testOp3.Opaque, true) != nil {
suite.T().Fatalf("The op should not have been there")
}
// Drain
rd.Add(testOp2)
rd.Add(testOp1)
found1 := 0
found2 := 0
rd.Drain(func(op *memdQRequest) {
if op == testOp1 {
found1++
}
if op == testOp2 {
found2++
}
})
if found1 != 1 || found2 != 1 {
suite.T().Fatalf("Drain behaved incorrected")
}
}
gocbcore-10.2.3/memdopqueue.go 0000664 0000000 0000000 00000006366 14417540156 0016257 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"container/list"
"errors"
"fmt"
"sync"
"sync/atomic"
"unsafe"
)
var (
errOpQueueClosed = errors.New("queue is closed")
errOpQueueFull = errors.New("queue is full")
errAlreadyQueued = errors.New("request was already queued somewhere else")
)
type memdOpConsumer struct {
parent *memdOpQueue
isClosed bool
}
func (c *memdOpConsumer) Queue() *memdOpQueue {
return c.parent
}
func (c *memdOpConsumer) Pop() *memdQRequest {
return c.parent.pop(c)
}
func (c *memdOpConsumer) Close() {
c.parent.closeConsumer(c)
}
type memdOpQueue struct {
lock sync.Mutex
signal *sync.Cond
items *list.List
isOpen bool
}
func newMemdOpQueue() *memdOpQueue {
q := memdOpQueue{
isOpen: true,
items: list.New(),
}
q.signal = sync.NewCond(&q.lock)
return &q
}
// nolint: unused
func (q *memdOpQueue) debugString() string {
var outStr string
q.lock.Lock()
outStr += fmt.Sprintf("Num Items: %d\n", q.items.Len())
outStr += fmt.Sprintf("Is Open: %t", q.isOpen)
q.lock.Unlock()
return outStr
}
func (q *memdOpQueue) Remove(req *memdQRequest) bool {
q.lock.Lock()
if !atomic.CompareAndSwapPointer(&req.queuedWith, unsafe.Pointer(q), nil) {
q.lock.Unlock()
return false
}
for e := q.items.Front(); e != nil; e = e.Next() {
if e.Value.(*memdQRequest) == req {
q.items.Remove(e)
break
}
}
q.lock.Unlock()
return true
}
func (q *memdOpQueue) Push(req *memdQRequest, maxItems int) error {
q.lock.Lock()
if !q.isOpen {
q.lock.Unlock()
return errOpQueueClosed
}
if maxItems > 0 && q.items.Len() >= maxItems {
q.lock.Unlock()
return errOpQueueFull
}
if !atomic.CompareAndSwapPointer(&req.queuedWith, nil, unsafe.Pointer(q)) {
q.lock.Unlock()
return errAlreadyQueued
}
if req.isCancelled() {
atomic.CompareAndSwapPointer(&req.queuedWith, unsafe.Pointer(q), nil)
q.lock.Unlock()
return errRequestCanceled
}
q.items.PushBack(req)
q.lock.Unlock()
q.signal.Broadcast()
return nil
}
func (q *memdOpQueue) Consumer() *memdOpConsumer {
return &memdOpConsumer{
parent: q,
isClosed: false,
}
}
func (q *memdOpQueue) closeConsumer(c *memdOpConsumer) {
q.lock.Lock()
c.isClosed = true
q.lock.Unlock()
q.signal.Broadcast()
}
func (q *memdOpQueue) pop(c *memdOpConsumer) *memdQRequest {
q.lock.Lock()
for q.isOpen && !c.isClosed && q.items.Len() == 0 {
q.signal.Wait()
}
if !q.isOpen || c.isClosed {
q.lock.Unlock()
return nil
}
e := q.items.Front()
q.items.Remove(e)
req, ok := e.Value.(*memdQRequest)
if !ok {
logErrorf("Encountered incorrect type in memdOpQueue")
return q.pop(c)
}
atomic.CompareAndSwapPointer(&req.queuedWith, unsafe.Pointer(q), nil)
q.lock.Unlock()
return req
}
type drainCallback func(*memdQRequest)
func (q *memdOpQueue) Drain(cb drainCallback) {
q.lock.Lock()
if q.isOpen {
logErrorf("Attempted to Drain open memdOpQueue, ignoring")
q.lock.Unlock()
return
}
for e := q.items.Front(); e != nil; e = e.Next() {
req, ok := e.Value.(*memdQRequest)
if !ok {
logErrorf("Encountered incorrect type in memdOpQueue")
continue
}
atomic.CompareAndSwapPointer(&req.queuedWith, unsafe.Pointer(q), nil)
cb(req)
}
q.lock.Unlock()
}
func (q *memdOpQueue) Close() {
q.lock.Lock()
q.isOpen = false
q.lock.Unlock()
q.signal.Broadcast()
}
gocbcore-10.2.3/memdpipeline.go 0000664 0000000 0000000 00000012404 14417540156 0016367 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"fmt"
"sync"
)
var (
errPipelineClosed = errors.New("pipeline has been closed")
errPipelineFull = errors.New("pipeline is too full")
)
type memdGetClientFn func(cancelSig <-chan struct{}) (*memdClient, error)
type memdPipeline struct {
address string
getClientFn memdGetClientFn
maxItems int
queue *memdOpQueue
maxClients int
clients []*memdPipelineClient
clientsLock sync.Mutex
isSeedNode bool
}
func newPipeline(endpoint routeEndpoint, maxClients, maxItems int, getClientFn memdGetClientFn) *memdPipeline {
return &memdPipeline{
address: endpoint.Address,
getClientFn: getClientFn,
maxClients: maxClients,
maxItems: maxItems,
queue: newMemdOpQueue(),
isSeedNode: endpoint.IsSeedNode,
}
}
func newDeadPipeline(maxItems int) *memdPipeline {
return newPipeline(routeEndpoint{}, 0, maxItems, nil)
}
// nolint: unused
func (pipeline *memdPipeline) debugString() string {
var outStr string
if pipeline.address != "" {
outStr += fmt.Sprintf("Address: %s\n", pipeline.address)
outStr += fmt.Sprintf("Max Clients: %d\n", pipeline.maxClients)
outStr += fmt.Sprintf("Num Clients: %d\n", len(pipeline.clients))
outStr += fmt.Sprintf("Max Items: %d\n", pipeline.maxItems)
} else {
outStr += "Dead-Server Queue\n"
}
outStr += "Op Queue:\n"
outStr += reindentLog(" ", pipeline.queue.debugString())
return outStr
}
func (pipeline *memdPipeline) IsSeedNode() bool {
return pipeline.isSeedNode
}
func (pipeline *memdPipeline) Clients() []*memdPipelineClient {
pipeline.clientsLock.Lock()
defer pipeline.clientsLock.Unlock()
return pipeline.clients
}
func (pipeline *memdPipeline) Address() string {
return pipeline.address
}
func (pipeline *memdPipeline) StartClients() {
pipeline.clientsLock.Lock()
defer pipeline.clientsLock.Unlock()
for len(pipeline.clients) < pipeline.maxClients {
client := newMemdPipelineClient(pipeline)
pipeline.clients = append(pipeline.clients, client)
go client.Run()
}
}
func (pipeline *memdPipeline) sendRequest(req *memdQRequest, maxItems int) error {
err := pipeline.queue.Push(req, maxItems)
if err == errOpQueueClosed {
return errPipelineClosed
} else if err == errOpQueueFull {
return errPipelineFull
} else if err != nil {
return err
}
return nil
}
func (pipeline *memdPipeline) RequeueRequest(req *memdQRequest) error {
return pipeline.sendRequest(req, 0)
}
func (pipeline *memdPipeline) SendRequest(req *memdQRequest) error {
return pipeline.sendRequest(req, pipeline.maxItems)
}
// Performs a takeover of another pipeline. Note that this does not
// take over the requests queued in the old pipeline, and those must
// be drained and processed separately.
func (pipeline *memdPipeline) Takeover(oldPipeline *memdPipeline) {
if oldPipeline.address != pipeline.address {
logErrorf("Attempted pipeline takeover for differing address")
// We try to 'gracefully' error here by resolving all the requests as
// errors, but allowing the application to continue.
err := oldPipeline.Close()
if err != nil {
// Log and continue with this non-fatal error.
logDebugf("Failed to shutdown old pipeline (%s)", err)
}
// Drain all the requests as an internal error so they are not lost
oldPipeline.Drain(func(req *memdQRequest) {
req.tryCallback(nil, errCliInternalError)
})
return
}
// Migrate all the clients to the new pipeline
oldPipeline.clientsLock.Lock()
clients := oldPipeline.clients
oldPipeline.clients = nil
oldPipeline.clientsLock.Unlock()
pipeline.clientsLock.Lock()
pipeline.clients = clients
for _, client := range pipeline.clients {
client.ReassignTo(pipeline)
}
pipeline.clientsLock.Unlock()
// Shut down the old pipelines queue, this will force all the
// clients to 'refresh' their consumer, and pick up the new
// pipeline queue from the new pipeline. This will also block
// any writers from sending new requests here if they have an
// out of date route config.
oldPipeline.queue.Close()
}
func (pipeline *memdPipeline) GracefulClose() []*memdClient {
// Shut down all the clients
pipeline.clientsLock.Lock()
clients := pipeline.clients
pipeline.clients = nil
pipeline.clientsLock.Unlock()
var memdClients []*memdClient
for _, pipecli := range clients {
client := pipecli.CloseAndTakeClient()
logDebugf("Pipeline %s/%p taking memdclient %p from client %p", pipeline.address, pipeline, client, pipecli)
if client != nil {
memdClients = append(memdClients, client)
}
}
// Kill the queue, forcing everyone to stop
pipeline.queue.Close()
return memdClients
}
func (pipeline *memdPipeline) Close() error {
// Shut down all the clients
pipeline.clientsLock.Lock()
clients := pipeline.clients
pipeline.clients = nil
pipeline.clientsLock.Unlock()
hadErrors := false
for _, pipecli := range clients {
client := pipecli.CloseAndTakeClient()
if client != nil {
err := client.Close()
if err != nil {
logErrorf("failed to shutdown memdclient: %s", err)
hadErrors = true
}
// Wait for the client to finish closing.
<-client.CloseNotify()
}
}
// Kill the queue, forcing everyone to stop
pipeline.queue.Close()
if hadErrors {
return errCliInternalError
}
return nil
}
func (pipeline *memdPipeline) Drain(cb func(*memdQRequest)) {
pipeline.queue.Drain(cb)
}
gocbcore-10.2.3/memdpipelineclient.go 0000664 0000000 0000000 00000022523 14417540156 0017571 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"io"
"sync"
"sync/atomic"
)
type clientWait struct {
client *memdClient
err error
}
type memdPipelineClient struct {
parent *memdPipeline
address string
client *memdClient
consumer *memdOpConsumer
lock sync.Mutex
closedSig chan struct{}
clientTakenSig chan struct{}
cancelDialSig chan struct{}
state uint32
connectError error
}
func newMemdPipelineClient(parent *memdPipeline) *memdPipelineClient {
return &memdPipelineClient{
parent: parent,
address: parent.address,
closedSig: make(chan struct{}),
clientTakenSig: make(chan struct{}),
cancelDialSig: make(chan struct{}),
state: uint32(EndpointStateDisconnected),
}
}
func (pipecli *memdPipelineClient) State() EndpointState {
return EndpointState(atomic.LoadUint32(&pipecli.state))
}
func (pipecli *memdPipelineClient) Error() error {
pipecli.lock.Lock()
defer pipecli.lock.Unlock()
return pipecli.connectError
}
func (pipecli *memdPipelineClient) ReassignTo(parent *memdPipeline) {
pipecli.lock.Lock()
pipecli.parent = parent
oldConsumer := pipecli.consumer
pipecli.consumer = nil
pipecli.lock.Unlock()
if oldConsumer != nil {
oldConsumer.Close()
}
}
func (pipecli *memdPipelineClient) ioLoop(client *memdClient) {
pipecli.lock.Lock()
if pipecli.parent == nil {
logDebugf("Pipeline client ioLoop started with no parent pipeline")
pipecli.lock.Unlock()
err := client.Close()
if err != nil {
logErrorf("Failed to close client for shut down ioLoop (%s)", err)
}
return
}
pipecli.client = client
pipecli.lock.Unlock()
killSig := make(chan struct{})
// This goroutine is responsible for monitoring the client and handling
// the cleanup whenever it shuts down. All cases of the client being
// shut down flow through this goroutine, even cases where we may already
// be aware that the client is shutdown, outside this scope.
go func() {
logDebugf("Pipeline client `%s/%p` client watcher starting...", pipecli.address, pipecli)
select {
case <-client.CloseNotify():
logDebugf("Pipeline client `%s/%p` client died", pipecli.address, pipecli)
case <-pipecli.clientTakenSig:
logDebugf("Pipeline client `%s/%p` client taken", pipecli.address, pipecli)
}
pipecli.lock.Lock()
pipecli.client = nil
activeConsumer := pipecli.consumer
pipecli.consumer = nil
pipecli.lock.Unlock()
logDebugf("Pipeline client `%s/%p` closing consumer %p", pipecli.address, pipecli, activeConsumer)
// If we have a consumer, we need to close it to signal the loop below that
// something has happened. If there is no consumer, we don't need to signal
// as the loop below will already be in the process of fetching a new one,
// where it will inevitably detect the problem.
if activeConsumer != nil {
activeConsumer.Close()
}
killSig <- struct{}{}
}()
logDebugf("Pipeline client `%s/%p` IO loop starting...", pipecli.address, pipecli)
var localConsumer *memdOpConsumer
for {
if localConsumer == nil {
logDebugf("Pipeline client `%s/%p` fetching new consumer", pipecli.address, pipecli)
pipecli.lock.Lock()
if pipecli.consumer != nil {
// If we still have an active consumer, lets close it to make room for the new one
pipecli.consumer.Close()
pipecli.consumer = nil
}
if pipecli.client == nil {
// The client has disconnected from the server, this only occurs AFTER the watcher
// goroutine running above has detected the client is closed and has cleaned it up.
pipecli.lock.Unlock()
break
}
if pipecli.parent == nil {
// This pipelineClient has been shut down
logDebugf("Pipeline client `%s/%p` found no parent pipeline", pipecli.address, pipecli)
pipecli.lock.Unlock()
break
}
// Fetch a new consumer to use for this iteration
localConsumer = pipecli.parent.queue.Consumer()
pipecli.consumer = localConsumer
pipecli.lock.Unlock()
}
req := localConsumer.Pop()
if req == nil {
// Set the local consumer to null, this will force our normal logic to run
// which will clean up the original consumer and then attempt to acquire a
// new one if we are not being cleaned up. This is a minor code-optimization
// to avoid having to do a lock/unlock just to lock above anyways. It does
// have the downside of not being able to detect where we've looped around
// in error though.
localConsumer = nil
continue
}
err := client.SendRequest(req)
if err != nil {
logDebugf("Pipeline client `%s/%p` encountered a socket write error: %v", pipecli.address, pipecli, err)
if !errors.Is(err, io.EOF) && !errors.Is(err, ErrMemdClientClosed) {
// If we errored the write, and the client was not already closed,
// lets go ahead and close it. This will trigger the shutdown
// logic via the client watcher above. If the socket error was EOF
// we already did shut down, and the watcher should already be
// cleaning up. If the error was ErrMemdClientClosed then client either
// did shutdown or is gracefully shutting down.
err := client.Close()
if err != nil {
logErrorf("Pipeline client `%s/%p` failed to shut down errored client socket (%s)", pipecli.address, pipecli, err)
}
}
// Send this request upwards to be processed by the higher level processor
shortCircuited, routeErr := client.postErrHandler(nil, req, err)
if !shortCircuited {
client.CancelRequest(req, err)
req.tryCallback(nil, routeErr)
break
}
// Stop looping
break
}
}
atomic.StoreUint32(&pipecli.state, uint32(EndpointStateDisconnecting))
// We must wait for the close wait goroutine to die as well before we can continue.
<-killSig
logDebugf("Pipeline client `%s/%p` received client shutdown notification", pipecli.address, pipecli)
}
func (pipecli *memdPipelineClient) Run() {
for {
logDebugf("Pipeline Client `%s/%p` preparing for new client loop", pipecli.address, pipecli)
atomic.StoreUint32(&pipecli.state, uint32(EndpointStateConnecting))
pipecli.lock.Lock()
pipeline := pipecli.parent
pipecli.lock.Unlock()
if pipeline == nil {
// If our pipeline is nil, it indicates that we need to shut down.
logDebugf("Pipeline Client `%s/%p` is shutting down", pipecli.address, pipecli)
break
}
logDebugf("Pipeline Client `%s/%p` retrieving new client connection for parent %p", pipecli.address, pipecli, pipeline)
wait := make(chan clientWait, 1)
go func() {
client, err := pipeline.getClientFn(pipecli.cancelDialSig)
wait <- clientWait{
client: client,
err: err,
}
}()
cli := <-wait
if cli.err != nil {
atomic.StoreUint32(&pipecli.state, uint32(EndpointStateDisconnected))
pipecli.lock.Lock()
if pipecli.parent != nil {
// If we know that we're shutting then don't log the error, it isn't unexpected.
logWarnf("Pipeline Client %p failed to bootstrap: %s", pipecli, cli.err)
}
pipecli.connectError = cli.err
pipecli.lock.Unlock()
continue
}
pipecli.lock.Lock()
pipecli.connectError = nil
pipecli.lock.Unlock()
atomic.StoreUint32(&pipecli.state, uint32(EndpointStateConnected))
// Runs until the connection has died (for whatever reason)
logDebugf("Pipeline Client `%s/%p` starting new client loop for %p", pipecli.address, pipecli, cli.client)
pipecli.ioLoop(cli.client)
}
// Lets notify anyone who is watching that we are now shut down
close(pipecli.closedSig)
}
// CloseAndTakeClient will close this pipeline client, yielding the memdClient. Note that this method will not wait for
// everything to be cleaned up before returning.
func (pipecli *memdPipelineClient) CloseAndTakeClient() *memdClient {
logDebugf("Pipeline Client `%s/%p` received close request", pipecli.address, pipecli)
atomic.StoreUint32(&pipecli.state, uint32(EndpointStateDisconnecting))
// To shut down the client, we remove our reference to the parent. This
// causes our ioLoop see that we are being shut down and perform cleanup
// before exiting.
pipecli.lock.Lock()
pipecli.parent = nil
activeConsumer := pipecli.consumer
pipecli.consumer = nil
client := pipecli.client
pipecli.client = nil
pipecli.lock.Unlock()
close(pipecli.clientTakenSig)
logDebugf("Pipeline client `%s/%p` closing consumer %p", pipecli.address, pipecli, activeConsumer)
// If we have a consumer, we need to close it to signal the loop below that
// something has happened. If there is no consumer, we don't need to signal
// as the loop below will already be in the process of fetching a new one,
// where it will inevitably detect the problem.
if activeConsumer != nil {
activeConsumer.Close()
}
// We might be currently waiting for a new client to be dialled, in which we need to abandon that wait so that
// it does not block our shutdown.
close(pipecli.cancelDialSig)
// If we have an active consumer, we need to close it to cause the running
// ioLoop to unpause and pick up that our parent has been removed. Note
// that in some cases, we might not have an active consumer. This means
// that the ioLoop is about to try and fetch one, finding the missing
// parent in doing so.
if activeConsumer != nil {
activeConsumer.Close()
}
// Lets wait till the ioLoop has shut everything down before returning.
<-pipecli.closedSig
atomic.StoreUint32(&pipecli.state, uint32(EndpointStateDisconnected))
logDebugf("Pipeline Client `%s/%p` has exited", pipecli.address, pipecli)
return client
}
gocbcore-10.2.3/memdqpackets.go 0000664 0000000 0000000 00000017710 14417540156 0016402 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"fmt"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/couchbase/gocbcore/v10/memd"
)
// The data for a response from a server. This includes the
// packets data along with some useful meta-data related to
// the response.
type memdQResponse struct {
*memd.Packet
// remoteAddr and sourceAddr are opposite to what may be expected here, to reflect that this is the response.
remoteAddr string
sourceAddr string
sourceConnID string
}
type callback func(*memdQResponse, *memdQRequest, error)
// The data for a request that can be queued with a memdqueueconn,
// and can potentially be rerouted to multiple servers due to
// configuration changes.
type memdQRequest struct {
memd.Packet
// Static routing properties
ReplicaIdx int
Callback callback
Persistent bool
// This tracks when the request was dispatched so that we can
// properly prioritize older requests to try and meet timeout
// requirements.
dispatchTime time.Time
// This stores a pointer to the server that currently own
// this request. This allows us to remove it from that list
// whenever the request is cancelled.
queuedWith unsafe.Pointer
// This stores a pointer to the opList that currently is holding
// this request. This allows us to remove it form that list
// whenever the request is cancelled
waitingIn unsafe.Pointer
// This keeps track of whether the request has been 'completed'
// which is synonymous with the callback having been invoked.
// This is an integer to allow us to atomically control it.
isCompleted uint32
// This is used to lock access to the request when processing
// a timeout, a response or spans
processingLock sync.Mutex
// This stores the number of times that the item has been
// retried. It is used for various non-linear retry
// algorithms.
retryCount uint32
// This is used to determine what, if any, retry strategy to use
// when deciding whether to retry the request and calculating
// any back-off time period.
RetryStrategy RetryStrategy
// This is the set of reasons why this request has been retried.
retryReasons []RetryReason
// This is used to lock access to the request when processing
// retry reasons or attempts.
retryLock sync.Mutex
// This is the timer which is used for cancellation of the request when deadlines are used.
timer atomic.Value
// This stores a memdQRequestConnInfo value which is used to track connection information
// for the request.
connInfo atomic.Value
RootTraceContext RequestSpanContext
cmdTraceSpan RequestSpan
netTraceSpan RequestSpan
CollectionName string
ScopeName string
resourceUnitsLock sync.Mutex
resourceUnits *ResourceUnitResult
}
type memdQRequestConnInfo struct {
lastDispatchedTo string
lastDispatchedFrom string
lastConnectionID string
}
func (req *memdQRequest) AddResourceUnits(readUnitsFrame *memd.ReadUnitsFrame, writeUnitsFrame *memd.WriteUnitsFrame) {
if readUnitsFrame == nil && writeUnitsFrame == nil {
return
}
req.resourceUnitsLock.Lock()
if req.resourceUnits == nil {
req.resourceUnits = &ResourceUnitResult{}
}
if readUnitsFrame != nil {
req.resourceUnits.ReadUnits += readUnitsFrame.ReadUnits
}
if writeUnitsFrame != nil {
req.resourceUnits.WriteUnits += writeUnitsFrame.WriteUnits
}
req.resourceUnitsLock.Unlock()
}
func (req *memdQRequest) AddResourceUnitsFromUnitResult(unit *ResourceUnitResult) {
if unit == nil {
return
}
req.resourceUnitsLock.Lock()
if req.resourceUnits == nil {
req.resourceUnits = &ResourceUnitResult{}
}
req.resourceUnits.ReadUnits += unit.ReadUnits
req.resourceUnits.WriteUnits += unit.WriteUnits
req.resourceUnitsLock.Unlock()
}
func (req *memdQRequest) ResourceUnits() *ResourceUnitResult {
req.resourceUnitsLock.Lock()
if req.resourceUnits == nil {
req.resourceUnitsLock.Unlock()
return nil
}
units := &ResourceUnitResult{
ReadUnits: req.resourceUnits.ReadUnits,
WriteUnits: req.resourceUnits.WriteUnits,
}
req.resourceUnitsLock.Unlock()
return units
}
func (req *memdQRequest) RetryAttempts() uint32 {
req.retryLock.Lock()
defer req.retryLock.Unlock()
return req.retryCount
}
func (req *memdQRequest) RetryReasons() []RetryReason {
req.retryLock.Lock()
defer req.retryLock.Unlock()
return req.retryReasons
}
// Retries is here because we're locked into a publically exposed interface for RetryAttempts/RetryReasons.
// This function allows us to internally get count and reasons together preventing any races causing the count and
// reasons to mismatch.
func (req *memdQRequest) Retries() (uint32, []RetryReason) {
req.retryLock.Lock()
defer req.retryLock.Unlock()
return req.retryCount, req.retryReasons
}
func (req *memdQRequest) retryStrategy() RetryStrategy {
return req.RetryStrategy
}
func (req *memdQRequest) Identifier() string {
return fmt.Sprintf("%d", atomic.LoadUint32(&req.Opaque))
}
func (req *memdQRequest) Idempotent() bool {
_, ok := idempotentOps[req.Command]
return ok
}
func (req *memdQRequest) ConnectionInfo() memdQRequestConnInfo {
p := req.connInfo.Load()
if p == nil {
return memdQRequestConnInfo{}
}
return p.(memdQRequestConnInfo)
}
func (req *memdQRequest) SetConnectionInfo(info memdQRequestConnInfo) {
req.connInfo.Store(info)
}
func (req *memdQRequest) SetTimer(t *time.Timer) {
req.timer.Store(t)
}
func (req *memdQRequest) Timer() *time.Timer {
t := req.timer.Load()
if t == nil {
return nil
}
return t.(*time.Timer)
}
func (req *memdQRequest) recordRetryAttempt(retryReason RetryReason) {
req.retryLock.Lock()
defer req.retryLock.Unlock()
req.retryCount++
found := false
for i := 0; i < len(req.retryReasons); i++ {
if req.retryReasons[i] == retryReason {
found = true
break
}
}
// if idx is out of the range of retryReasons then it wasn't found.
if !found {
req.retryReasons = append(req.retryReasons, retryReason)
}
}
func (req *memdQRequest) tryCallback(resp *memdQResponse, err error) {
if t := req.Timer(); t != nil {
t.Stop()
}
if req.Persistent {
if err != nil {
if req.internalCancel(err) {
req.Callback(resp, req, err)
}
} else {
if atomic.LoadUint32(&req.isCompleted) == 0 {
req.Callback(resp, req, err)
}
}
} else {
if atomic.SwapUint32(&req.isCompleted, 1) == 0 {
req.Callback(resp, req, err)
}
}
}
func (req *memdQRequest) isCancelled() bool {
return atomic.LoadUint32(&req.isCompleted) != 0
}
func (req *memdQRequest) internalCancel(err error) bool {
req.processingLock.Lock()
if atomic.SwapUint32(&req.isCompleted, 1) != 0 {
// Someone already completed this request
req.processingLock.Unlock()
return false
}
t := req.Timer()
if t != nil {
// This timer might have already fired and that's how we got here, however we might have also got here
// via other means so we should always try to stop it.
t.Stop()
}
queuedWith := (*memdOpQueue)(atomic.LoadPointer(&req.queuedWith))
if queuedWith != nil {
queuedWith.Remove(req)
}
var localAddr string
var remoteAddr string
waitingIn := (*memdClient)(atomic.LoadPointer(&req.waitingIn))
if waitingIn != nil {
waitingIn.CancelRequest(req, err)
localAddr = waitingIn.LocalAddress()
remoteAddr = waitingIn.Address()
}
cancelReqTrace(req, localAddr, remoteAddr)
req.processingLock.Unlock()
return true
}
func (req *memdQRequest) cancelWithCallback(err error) {
// Try to perform the cancellation, if it succeeds, we call the
// callback immediately on the users behalf.
if req.internalCancel(err) {
req.Callback(nil, req, err)
}
}
func (req *memdQRequest) cancelWithCallbackAndFinishTracer(err error, tracer *opTelemetryHandler) {
// Try to perform the cancellation, if it succeeds, we call the
// callback immediately on the users behalf.
// Only if cancel succeeds we also finish the tracer.
if req.internalCancel(err) {
tracer.Finish()
req.Callback(nil, req, err)
}
}
func (req *memdQRequest) Cancel() {
// Try to perform the cancellation, if it succeeds, we call the
// callback immediately on the users behalf.
err := errRequestCanceled
req.cancelWithCallback(err)
}
gocbcore-10.2.3/memdqsorter.go 0000664 0000000 0000000 00000000504 14417540156 0016257 0 ustar 00root root 0000000 0000000 package gocbcore
type memdQRequestSorter []*memdQRequest
func (list memdQRequestSorter) Len() int {
return len(list)
}
func (list memdQRequestSorter) Less(i, j int) bool {
return list[i].dispatchTime.Before(list[j].dispatchTime)
}
func (list memdQRequestSorter) Swap(i, j int) {
list[i], list[j] = list[j], list[i]
}
gocbcore-10.2.3/metrics.go 0000664 0000000 0000000 00000000737 14417540156 0015373 0 ustar 00root root 0000000 0000000 package gocbcore
// Meter handles metrics information for SDK operations.
type Meter interface {
Counter(name string, tags map[string]string) (Counter, error)
ValueRecorder(name string, tags map[string]string) (ValueRecorder, error)
}
// Counter is used for incrementing a synchronous count metric.
type Counter interface {
IncrementBy(num uint64)
}
// ValueRecorder is used for grouping synchronous count metrics.
type ValueRecorder interface {
RecordValue(val uint64)
}
gocbcore-10.2.3/metrics_test.go 0000664 0000000 0000000 00000004504 14417540156 0016426 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"sync"
"sync/atomic"
)
// noopMeter is a Meter implementation which performs no metrics operations.
type noopMeter struct {
}
var (
defaultNoopCounter = &noopCounter{}
defaultNoopValueRecorder = &noopValueRecorder{}
)
// Counter is used for incrementing a synchronous count metric.
func (nm noopMeter) Counter(name string, tags map[string]string) (Counter, error) {
return defaultNoopCounter, nil
}
// ValueRecorder is used for grouping synchronous count metrics.
func (nm noopMeter) ValueRecorder(name string, tags map[string]string) (ValueRecorder, error) {
return defaultNoopValueRecorder, nil
}
type noopCounter struct{}
func (bc *noopCounter) IncrementBy(num uint64) {
}
type noopValueRecorder struct{}
func (bc *noopValueRecorder) RecordValue(val uint64) {
}
type testCounter struct {
count uint64
}
func (tc *testCounter) IncrementBy(val uint64) {
atomic.AddUint64(&tc.count, val)
}
type testValueRecorder struct {
values []uint64
lock sync.Mutex
}
func (tvr *testValueRecorder) RecordValue(val uint64) {
tvr.lock.Lock()
tvr.values = append(tvr.values, val)
tvr.lock.Unlock()
}
type testMeter struct {
lock sync.Mutex
counters map[string]*testCounter
recorders map[string]*testValueRecorder
}
func newTestMeter() *testMeter {
return &testMeter{
counters: make(map[string]*testCounter),
recorders: make(map[string]*testValueRecorder),
}
}
func (tm *testMeter) Reset() {
tm.lock.Lock()
tm.counters = make(map[string]*testCounter)
tm.recorders = make(map[string]*testValueRecorder)
tm.lock.Unlock()
}
func (tm *testMeter) Counter(name string, tags map[string]string) (Counter, error) {
key := tags["db.operation"]
tm.lock.Lock()
counter := tm.counters[key]
if counter == nil {
counter = &testCounter{}
tm.counters[key] = counter
}
tm.lock.Unlock()
return counter, nil
}
func (tm *testMeter) ValueRecorder(name string, tags map[string]string) (ValueRecorder, error) {
key := tags["db.couchbase.service"]
if op, ok := tags["db.operation"]; ok {
key = key + ":" + op
}
tm.lock.Lock()
recorder := tm.recorders[key]
if recorder == nil {
recorder = &testValueRecorder{}
tm.recorders[key] = recorder
}
tm.lock.Unlock()
return recorder, nil
}
func makeMetricsKey(service, op string) string {
key := service
if op != "" {
key = key + ":" + op
}
return key
}
gocbcore-10.2.3/mock_configManager_test.go 0000664 0000000 0000000 00000001114 14417540156 0020523 0 ustar 00root root 0000000 0000000 // Code generated by mockery v1.0.0. DO NOT EDIT.
package gocbcore
import mock "github.com/stretchr/testify/mock"
// mockConfigManager is an autogenerated mock type for the configManager type
type mockConfigManager struct {
mock.Mock
}
// AddConfigWatcher provides a mock function with given fields: watcher
func (_m *mockConfigManager) AddConfigWatcher(watcher routeConfigWatcher) {
_m.Called(watcher)
}
// RemoveConfigWatcher provides a mock function with given fields: watcher
func (_m *mockConfigManager) RemoveConfigWatcher(watcher routeConfigWatcher) {
_m.Called(watcher)
}
gocbcore-10.2.3/mock_dispatcher_test.go 0000664 0000000 0000000 00000005155 14417540156 0020122 0 ustar 00root root 0000000 0000000 // Code generated by mockery v1.0.0. DO NOT EDIT.
package gocbcore
import mock "github.com/stretchr/testify/mock"
// mockDispatcher is an autogenerated mock type for the dispatcher type
type mockDispatcher struct {
mock.Mock
}
// CollectionsEnabled provides a mock function with given fields:
func (_m *mockDispatcher) CollectionsEnabled() bool {
ret := _m.Called()
var r0 bool
if rf, ok := ret.Get(0).(func() bool); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// DispatchDirect provides a mock function with given fields: req
func (_m *mockDispatcher) DispatchDirect(req *memdQRequest) (PendingOp, error) {
ret := _m.Called(req)
var r0 PendingOp
if rf, ok := ret.Get(0).(func(*memdQRequest) PendingOp); ok {
r0 = rf(req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(PendingOp)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(*memdQRequest) error); ok {
r1 = rf(req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// DispatchDirectToAddress provides a mock function with given fields: req, pipeline
func (_m *mockDispatcher) DispatchDirectToAddress(req *memdQRequest, pipeline *memdPipeline) (PendingOp, error) {
ret := _m.Called(req, pipeline)
var r0 PendingOp
if rf, ok := ret.Get(0).(func(*memdQRequest, *memdPipeline) PendingOp); ok {
r0 = rf(req, pipeline)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(PendingOp)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(*memdQRequest, *memdPipeline) error); ok {
r1 = rf(req, pipeline)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// PipelineSnapshot provides a mock function with given fields:
func (_m *mockDispatcher) PipelineSnapshot() (*pipelineSnapshot, error) {
ret := _m.Called()
var r0 *pipelineSnapshot
if rf, ok := ret.Get(0).(func() *pipelineSnapshot); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*pipelineSnapshot)
}
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// RequeueDirect provides a mock function with given fields: req, isRetry
func (_m *mockDispatcher) RequeueDirect(req *memdQRequest, isRetry bool) {
_m.Called(req, isRetry)
}
// SetPostCompleteErrorHandler provides a mock function with given fields: handler
func (_m *mockDispatcher) SetPostCompleteErrorHandler(handler postCompleteErrorHandler) {
_m.Called(handler)
}
// SupportsCollections provides a mock function with given fields:
func (_m *mockDispatcher) SupportsCollections() bool {
ret := _m.Called()
var r0 bool
if rf, ok := ret.Get(0).(func() bool); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
gocbcore-10.2.3/mock_httpComponentInterface_test.go 0000664 0000000 0000000 00000001571 14417540156 0022455 0 ustar 00root root 0000000 0000000 // Code generated by mockery v1.0.0. DO NOT EDIT.
package gocbcore
import mock "github.com/stretchr/testify/mock"
// mockHttpComponentInterface is an autogenerated mock type for the httpComponentInterface type
type mockHttpComponentInterface struct {
mock.Mock
}
// DoInternalHTTPRequest provides a mock function with given fields: req, skipConfigCheck
func (_m *mockHttpComponentInterface) DoInternalHTTPRequest(req *httpRequest, skipConfigCheck bool) (*HTTPResponse, error) {
ret := _m.Called(req, skipConfigCheck)
var r0 *HTTPResponse
if rf, ok := ret.Get(0).(func(*httpRequest, bool) *HTTPResponse); ok {
r0 = rf(req, skipConfigCheck)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*HTTPResponse)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(*httpRequest, bool) error); ok {
r1 = rf(req, skipConfigCheck)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
gocbcore-10.2.3/n1qlcomponent.go 0000664 0000000 0000000 00000047012 14417540156 0016520 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"context"
"encoding/json"
"errors"
"io/ioutil"
"strings"
"sync"
"sync/atomic"
"time"
)
// N1QLRowReader providers access to the rows of a n1ql query
type N1QLRowReader struct {
streamer *queryStreamer
endpoint string
statement string
statusCode int
}
// NextRow reads the next rows bytes from the stream
func (q *N1QLRowReader) NextRow() []byte {
return q.streamer.NextRow()
}
// Err returns any errors that occurred during streaming.
func (q N1QLRowReader) Err() error {
err := q.streamer.Err()
if err != nil {
return err
}
meta, metaErr := q.streamer.MetaData()
if metaErr != nil {
return metaErr
}
raw, descs, err := parseN1QLError(meta)
if err != nil {
return &N1QLError{
InnerError: err,
Errors: descs,
ErrorText: raw,
Statement: q.statement,
HTTPResponseCode: q.statusCode,
}
}
if len(descs) > 0 {
return &N1QLError{
InnerError: errors.New("query error"),
Errors: descs,
ErrorText: raw,
Statement: q.statement,
HTTPResponseCode: q.statusCode,
}
}
return nil
}
// MetaData fetches the non-row bytes streamed in the response.
func (q *N1QLRowReader) MetaData() ([]byte, error) {
return q.streamer.MetaData()
}
// Close immediately shuts down the connection
func (q *N1QLRowReader) Close() error {
return q.streamer.Close()
}
// PreparedName returns the name of the prepared statement created when using enhanced prepared statements.
// If the prepared name has not been seen on the stream then this will return an error.
// Volatile: This API is subject to change.
func (q N1QLRowReader) PreparedName() (string, error) {
val := q.streamer.EarlyMetadata("prepared")
if val == nil {
return "", wrapN1QLError(nil, "", errors.New("prepared name not found in metadata"), "", 0)
}
var name string
err := json.Unmarshal(val, &name)
if err != nil {
return "", wrapN1QLError(nil, "", errors.New("failed to parse prepared name"), "", 0)
}
return name, nil
}
// Endpoint returns the address that this query was run against.
// Internal: This should never be used and is not supported.
func (q *N1QLRowReader) Endpoint() string {
return q.endpoint
}
// N1QLQueryOptions represents the various options available for a n1ql query.
type N1QLQueryOptions struct {
Payload []byte
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
// Internal: This should never be used and is not supported.
Endpoint string
TraceContext RequestSpanContext
}
func wrapN1QLError(req *httpRequest, statement string, err error, errBody string, statusCode int) *N1QLError {
if err == nil {
err = errors.New("query error")
}
ierr := &N1QLError{
InnerError: err,
}
if req != nil {
ierr.Endpoint = req.Endpoint
ierr.ClientContextID = req.UniqueID
ierr.RetryAttempts = req.RetryAttempts()
ierr.RetryReasons = req.RetryReasons()
}
ierr.ErrorText = errBody
ierr.Statement = statement
ierr.HTTPResponseCode = statusCode
return ierr
}
type jsonN1QLError struct {
Code uint32 `json:"code"`
Msg string `json:"msg"`
Reason map[string]interface{} `json:"reason"`
Retry bool `json:"retry"`
}
type jsonN1QLErrorResponse struct {
Errors json.RawMessage
}
func extractN1QL12009Error(err N1QLErrorDesc) error {
if len(err.Reason) > 0 {
if code, ok := err.Reason["code"]; ok {
// sad panda
code = int(code.(float64))
if code == 12033 {
return errCasMismatch
} else if code == 17014 {
return errDocumentNotFound
} else if code == 17012 {
return errDocumentExists
}
}
return errDMLFailure
}
if strings.Contains(strings.ToLower(err.Message), "cas mismatch") {
return errCasMismatch
}
return errDMLFailure
}
func parseN1QLErrorResp(req *httpRequest, statement string, resp *HTTPResponse) *N1QLError {
var errorDescs []N1QLErrorDesc
var err error
var raw string
respBody, readErr := ioutil.ReadAll(resp.Body)
if readErr == nil {
raw, errorDescs, err = parseN1QLError(respBody)
}
errOut := wrapN1QLError(req, statement, err, raw, resp.StatusCode)
errOut.Errors = errorDescs
return errOut
}
func parseN1QLError(respBody []byte) (string, []N1QLErrorDesc, error) {
var err error
var errorDescs []N1QLErrorDesc
var rawRespParse jsonN1QLErrorResponse
parseErr := json.Unmarshal(respBody, &rawRespParse)
if parseErr != nil {
return "", nil, nil
}
var respParse []jsonN1QLError
parseErr = json.Unmarshal(rawRespParse.Errors, &respParse)
if parseErr == nil {
for _, jsonErr := range respParse {
errorDescs = append(errorDescs, N1QLErrorDesc{
Code: jsonErr.Code,
Message: jsonErr.Msg,
Reason: jsonErr.Reason,
Retry: jsonErr.Retry,
})
}
}
if len(errorDescs) >= 1 {
firstErr := errorDescs[0]
errCode := firstErr.Code
errCodeGroup := errCode / 1000
if errCodeGroup == 4 {
err = errPlanningFailure
}
if errCodeGroup == 5 {
err = errInternalServerFailure
}
if errCodeGroup == 12 || errCodeGroup == 14 && errCode != 12004 && errCode != 12016 {
err = errIndexFailure
}
if errCode == 4040 || errCode == 4050 || errCode == 4060 || errCode == 4070 || errCode == 4080 || errCode == 4090 {
err = errPreparedStatementFailure
}
if errCode == 1191 || errCode == 1192 || errCode == 1193 || errCode == 1194 {
err = errRateLimitedFailure
}
if errCode == 5000 && strings.Contains(strings.ToLower(firstErr.Message),
"limit for number of indexes that can be created per scope has been reached") {
err = errQuotaLimitedFailure
}
if errCode == 1080 {
err = errUnambiguousTimeout
}
if errCode == 3000 {
err = errParsingFailure
}
if errCode == 12009 {
err = extractN1QL12009Error(firstErr)
}
if errCode == 13014 {
err = errAuthenticationFailure
}
if errCode == 1197 {
err = wrapError(errFeatureNotAvailable, "this server requires that a query context be used for queries")
}
if errCodeGroup == 10 {
err = errAuthenticationFailure
}
}
var rawErrors string
if err == nil && len(rawRespParse.Errors) > 0 {
// Only populate if this is an error that we don't recognise.
rawErrors = string(rawRespParse.Errors)
}
return rawErrors, errorDescs, err
}
type n1qlQueryComponent struct {
httpComponent httpComponentInterface
cfgMgr configManager
tracer *tracerComponent
queryCache *n1qlQueryCache
enhancedPreparedSupported uint32
}
type n1qlQueryCache struct {
cache map[string]*n1qlQueryCacheEntry
cacheLock sync.RWMutex
}
func newN1qlQueryCache() *n1qlQueryCache {
return &n1qlQueryCache{
cache: make(map[string]*n1qlQueryCacheEntry),
}
}
func (cache *n1qlQueryCache) Invalidate() {
cache.cacheLock.Lock()
cache.cache = make(map[string]*n1qlQueryCacheEntry)
cache.cacheLock.Unlock()
}
func (cache *n1qlQueryCache) Put(statement string, entry *n1qlQueryCacheEntry) {
cache.cacheLock.Lock()
cache.cache[statement] = entry
cache.cacheLock.Unlock()
}
func (cache *n1qlQueryCache) Delete(statement string) {
cache.cacheLock.Lock()
delete(cache.cache, statement)
cache.cacheLock.Unlock()
}
func (cache *n1qlQueryCache) Get(statement string) *n1qlQueryCacheEntry {
cache.cacheLock.RLock()
entry := cache.cache[statement]
if entry == nil {
cache.cacheLock.RUnlock()
return nil
}
cached := *entry
cache.cacheLock.RUnlock()
return &cached
}
type n1qlQueryCacheEntry struct {
name string
encodedPlan string
}
type n1qlJSONPrepData struct {
EncodedPlan string `json:"encoded_plan"`
Name string `json:"name"`
}
func newN1QLQueryComponent(httpComponent httpComponentInterface, cfgMgr configManager, tracer *tracerComponent) *n1qlQueryComponent {
nqc := &n1qlQueryComponent{
httpComponent: httpComponent,
cfgMgr: cfgMgr,
queryCache: newN1qlQueryCache(),
tracer: tracer,
}
cfgMgr.AddConfigWatcher(nqc)
return nqc
}
func (nqc *n1qlQueryComponent) OnNewRouteConfig(cfg *routeConfig) {
if atomic.LoadUint32(&nqc.enhancedPreparedSupported) == 0 &&
cfg.ContainsClusterCapability(1, "n1ql", "enhancedPreparedStatements") {
logDebugf("Enabling enhanced prepared statement support")
// Once supported this can't be unsupported
nqc.queryCache.Invalidate()
atomic.StoreUint32(&nqc.enhancedPreparedSupported, 1)
}
}
// N1QLQuery executes a N1QL query
func (nqc *n1qlQueryComponent) N1QLQuery(opts N1QLQueryOptions, cb N1QLQueryCallback) (PendingOp, error) {
tracer := nqc.tracer.StartTelemeteryHandler(metricValueServiceQueryValue, "N1QLQuery",
opts.TraceContext)
var payloadMap map[string]interface{}
err := json.Unmarshal(opts.Payload, &payloadMap)
if err != nil {
tracer.Finish()
return nil, wrapN1QLError(nil, "", wrapError(err, "expected a JSON payload"), "", 0)
}
statement := getMapValueString(payloadMap, "statement", "")
clientContextID := getMapValueString(payloadMap, "client_context_id", "")
readOnly := getMapValueBool(payloadMap, "readonly", false)
ctx, cancel := context.WithCancel(context.Background())
ireq := &httpRequest{
Service: N1qlService,
Method: "POST",
Path: "/query/service",
IsIdempotent: readOnly,
UniqueID: clientContextID,
Deadline: opts.Deadline,
RetryStrategy: opts.RetryStrategy,
RootTraceContext: tracer.RootContext(),
Context: ctx,
CancelFunc: cancel,
User: opts.User,
Endpoint: opts.Endpoint,
}
go func() {
resp, err := nqc.execute(ireq, payloadMap, statement, time.Now())
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
tracer.Finish()
cb(resp, nil)
}()
return ireq, nil
}
// PreparedN1QLQuery executes a prepared N1QL query
func (nqc *n1qlQueryComponent) PreparedN1QLQuery(opts N1QLQueryOptions, cb N1QLQueryCallback) (PendingOp, error) {
tracer := nqc.tracer.StartTelemeteryHandler(metricValueServiceQueryValue, "PreparedN1QLQuery", opts.TraceContext)
ctx, cancel := context.WithCancel(context.Background())
parentReqForCancel := &httpRequest{
Context: ctx,
CancelFunc: cancel,
}
go func() {
res, err := nqc.executePrepared(ctx, cancel, tracer.RootContext(), opts)
if err != nil {
cancel()
tracer.Finish()
cb(nil, err)
return
}
tracer.Finish()
cb(res, nil)
}()
return parentReqForCancel, nil
}
func (nqc *n1qlQueryComponent) executePrepared(ctx context.Context, cancel context.CancelFunc,
traceCtx RequestSpanContext, opts N1QLQueryOptions) (*N1QLRowReader, error) {
start := time.Now()
var payloadMap map[string]interface{}
err := json.Unmarshal(opts.Payload, &payloadMap)
if err != nil {
return nil, wrapN1QLError(nil, "", wrapError(err, "expected a JSON payload"), "", 0)
}
statement := getMapValueString(payloadMap, "statement", "")
clientContextID := getMapValueString(payloadMap, "client_context_id", "")
readOnly := getMapValueBool(payloadMap, "readonly", false)
cachedStmt := nqc.queryCache.Get(statement)
enhanced := atomic.LoadUint32(&nqc.enhancedPreparedSupported) == 1
var req *httpRequest
if cachedStmt != nil {
// Attempt to execute our cached query plan
delete(payloadMap, "statement")
payloadMap["prepared"] = cachedStmt.name
if cachedStmt.encodedPlan != "" {
payloadMap["encoded_plan"] = cachedStmt.encodedPlan
}
req = &httpRequest{
Service: N1qlService,
Method: "POST",
Path: "/query/service",
IsIdempotent: readOnly,
UniqueID: clientContextID,
Deadline: opts.Deadline,
RetryStrategy: opts.RetryStrategy,
RootTraceContext: traceCtx,
Context: ctx,
CancelFunc: cancel,
User: opts.User,
Endpoint: opts.Endpoint,
}
results, err := nqc.execute(req, payloadMap, statement, start)
if err == nil {
return results, nil
}
retryErr := nqc.preparedStatementMaybeEvictAndRetry(req, err, start, statement)
if retryErr != nil {
return nil, retryErr
}
logDebugf("Prepared statement execution failed, will attempt reprepare: %v", err)
}
delete(payloadMap, "prepared")
delete(payloadMap, "encoded_plan")
payloadMap["statement"] = "PREPARE " + statement
if enhanced {
payloadMap["auto_execute"] = true
} else {
delete(payloadMap, "auto_execute")
}
if req == nil {
req = &httpRequest{
Service: N1qlService,
Method: "POST",
Path: "/query/service",
IsIdempotent: readOnly,
UniqueID: clientContextID,
Deadline: opts.Deadline,
RetryStrategy: opts.RetryStrategy,
RootTraceContext: traceCtx,
Context: ctx,
CancelFunc: cancel,
User: opts.User,
Endpoint: opts.Endpoint,
}
}
for {
var res *N1QLRowReader
var err error
if enhanced {
res, err = nqc.executeEnhPrepared(req, payloadMap, statement, start)
} else {
res, err = nqc.executeOldPrepared(req, payloadMap, statement, start)
}
if err == nil {
return res, nil
}
err = nqc.preparedStatementMaybeEvictAndRetry(req, err, start, statement)
if err != nil {
return nil, err
}
}
}
func (nqc *n1qlQueryComponent) preparedStatementMaybeEvictAndRetry(req *httpRequest, originalErr error, start time.Time,
statement string) error {
var err *N1QLError
if !errors.As(originalErr, &err) {
return originalErr
}
var retryReason RetryReason
if len(err.Errors) >= 1 {
firstErrDesc := err.Errors[0]
if firstErrDesc.Code == 4040 || firstErrDesc.Code == 4050 || firstErrDesc.Code == 4060 ||
firstErrDesc.Code == 4070 || firstErrDesc.Code == 4080 || firstErrDesc.Code == 4090 {
retryReason = QueryPreparedStatementFailureRetryReason
// If the error is because of a prepared statement issue then we need to evict the cache entry and reprepare.
nqc.queryCache.Delete(statement)
}
if retryReason == nil {
// n1qlErr is already wrapped here
return originalErr
}
shouldRetry, retryTime := retryOrchMaybeRetry(req, retryReason)
if !shouldRetry {
// n1qlErr is already wrapped here
return originalErr
}
select {
case <-time.After(time.Until(req.Deadline)):
err := &TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "N1QLQuery",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: req.retryReasons,
RetryAttempts: req.retryCount,
LastDispatchedTo: req.Endpoint,
}
return wrapN1QLError(req, statement, err, "", 0)
case <-time.After(time.Until(retryTime)):
return nil
}
}
return originalErr
}
func (nqc *n1qlQueryComponent) executeEnhPrepared(ireq *httpRequest, payloadMap map[string]interface{},
statement string, start time.Time) (*N1QLRowReader, error) {
cacheRes, err := nqc.execute(ireq, payloadMap, statement, start)
if err != nil {
return nil, err
}
preparedName, err := cacheRes.PreparedName()
if err != nil {
logWarnf("Failed to read prepared name from result: %s", err)
return cacheRes, nil
}
cachedStmt := &n1qlQueryCacheEntry{}
cachedStmt.name = preparedName
nqc.queryCache.Put(statement, cachedStmt)
return cacheRes, nil
}
func (nqc *n1qlQueryComponent) executeOldPrepared(ireq *httpRequest, payloadMap map[string]interface{}, statement string,
start time.Time) (*N1QLRowReader, error) {
delete(payloadMap, "prepared")
delete(payloadMap, "encoded_plan")
delete(payloadMap, "auto_execute")
prepStatement := "PREPARE " + statement
payloadMap["statement"] = prepStatement
cacheRes, err := nqc.execute(ireq, payloadMap, statement, start)
if err != nil {
return nil, err
}
b := cacheRes.NextRow()
if b == nil {
var n1qlError *N1QLError
meta, metaErr := cacheRes.MetaData()
if metaErr == nil {
raw, descs, err := parseN1QLError(meta)
if err != nil {
n1qlError = wrapN1QLError(ireq, statement, err, raw, 0)
n1qlError.Errors = descs
} else if len(descs) > 0 {
n1qlError = wrapN1QLError(ireq, statement, nil, raw, 0)
n1qlError.Errors = descs
}
}
if n1qlError == nil {
n1qlError = wrapN1QLError(ireq, statement, errCliInternalError, "", 0)
}
return nil, n1qlError
}
var prepData n1qlJSONPrepData
err = json.Unmarshal(b, &prepData)
if err != nil {
return nil, wrapN1QLError(ireq, statement, err, "", 0)
}
cachedStmt := &n1qlQueryCacheEntry{}
cachedStmt.name = prepData.Name
cachedStmt.encodedPlan = prepData.EncodedPlan
nqc.queryCache.Put(statement, cachedStmt)
// Attempt to execute our cached query plan
delete(payloadMap, "statement")
payloadMap["prepared"] = cachedStmt.name
payloadMap["encoded_plan"] = cachedStmt.encodedPlan
resp, err := nqc.execute(ireq, payloadMap, statement, start)
if err != nil {
return nil, err
}
return resp, nil
}
func (nqc *n1qlQueryComponent) execute(ireq *httpRequest, payloadMap map[string]interface{}, statementForErr string,
start time.Time) (*N1QLRowReader, error) {
for {
{
if !ireq.Deadline.IsZero() {
// Produce an updated payload with the appropriate timeout
timeoutLeft := time.Until(ireq.Deadline)
if timeoutLeft <= 0 {
err := &TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "N1QLQuery",
Opaque: ireq.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: ireq.retryReasons,
RetryAttempts: ireq.retryCount,
LastDispatchedTo: ireq.Endpoint,
}
return nil, wrapN1QLError(ireq, statementForErr, err, "", 0)
}
payloadMap["timeout"] = timeoutLeft.String()
}
newPayload, err := json.Marshal(payloadMap)
if err != nil {
return nil, wrapN1QLError(nil, "", wrapError(err, "failed to produce payload"), "", 0)
}
ireq.Body = newPayload
}
resp, err := nqc.httpComponent.DoInternalHTTPRequest(ireq, false)
if err != nil {
if errors.Is(err, ErrRequestCanceled) {
return nil, err
}
// execHTTPRequest will handle retrying due to in-flight socket close based
// on whether or not IsIdempotent is set on the httpRequest
return nil, wrapN1QLError(ireq, statementForErr, err, "", 0)
}
if resp.StatusCode != 200 {
n1qlErr := parseN1QLErrorResp(ireq, statementForErr, resp)
// Note that prepared statement error code retries are handled higher up.
var retryReason RetryReason
if len(n1qlErr.Errors) >= 1 {
firstErrDesc := n1qlErr.Errors[0]
// See MB-50643 for why this code check is here.
if firstErrDesc.Retry && firstErrDesc.Code != 12016 {
retryReason = QueryErrorRetryable
} else if strings.Contains(firstErrDesc.Message, "queryport.indexNotFound") {
retryReason = QueryIndexNotFoundRetryReason
}
}
if retryReason == nil {
// n1qlErr is already wrapped here
return nil, n1qlErr
}
shouldRetry, retryTime := retryOrchMaybeRetry(ireq, retryReason)
if !shouldRetry {
// n1qlErr is already wrapped here
return nil, n1qlErr
}
select {
case <-time.After(time.Until(retryTime)):
continue
case <-time.After(time.Until(ireq.Deadline)):
err := &TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "N1QLQuery",
Opaque: ireq.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: ireq.retryReasons,
RetryAttempts: ireq.retryCount,
LastDispatchedTo: ireq.Endpoint,
}
return nil, wrapN1QLError(ireq, statementForErr, err, "", 0)
}
}
streamer, err := newQueryStreamer(resp.Body, "results")
if err != nil {
respBody, readErr := ioutil.ReadAll(resp.Body)
if readErr != nil {
logDebugf("Failed to read response body: %v", readErr)
}
return nil, wrapN1QLError(ireq, statementForErr, err, string(respBody), resp.StatusCode)
}
return &N1QLRowReader{
streamer: streamer,
endpoint: resp.Endpoint,
statement: statementForErr,
statusCode: resp.StatusCode,
}, nil
}
}
gocbcore-10.2.3/n1qlcomponent_test.go 0000664 0000000 0000000 00000076767 14417540156 0017602 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"testing"
"time"
"github.com/stretchr/testify/mock"
)
type n1qlTestHelper struct {
TestName string
NumDocs int
QueryTestDocs *testDocs
suite *StandardTestSuite
}
func hlpRunQuery(t *testing.T, agent *AgentGroup, opts N1QLQueryOptions) ([][]byte, error) {
t.Helper()
resCh := make(chan *N1QLRowReader, 1)
errCh := make(chan error, 1)
_, err := agent.N1QLQuery(opts, func(reader *N1QLRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
return nil, err
}
var rows *N1QLRowReader
select {
case err := <-errCh:
return nil, err
case res := <-resCh:
rows = res
}
var rowBytes [][]byte
for {
row := rows.NextRow()
if row == nil {
break
}
rowBytes = append(rowBytes, row)
}
err = rows.Err()
return rowBytes, err
}
func hlpEnsurePrimaryIndex(t *testing.T, agent *AgentGroup, bucketName string) {
t.Helper()
payloadStr := fmt.Sprintf(`{"statement":"CREATE PRIMARY INDEX ON %s"}`, bucketName)
hlpRunQuery(t, agent, N1QLQueryOptions{
Payload: []byte(payloadStr),
Deadline: time.Now().Add(5000 * time.Millisecond),
})
}
func (nqh *n1qlTestHelper) testSetupN1ql(t *testing.T) {
agent := nqh.suite.DefaultAgent()
ag := nqh.suite.AgentGroup()
nqh.QueryTestDocs = makeTestDocs(t, agent, nqh.TestName, nqh.NumDocs)
hlpEnsurePrimaryIndex(t, ag, nqh.suite.BucketName)
}
func (nqh *n1qlTestHelper) testCleanupN1ql(t *testing.T) {
if nqh.QueryTestDocs != nil {
nqh.QueryTestDocs.Remove()
nqh.QueryTestDocs = nil
}
}
func (nqh *n1qlTestHelper) testN1QLBasic(t *testing.T) {
ag := nqh.suite.AgentGroup()
deadline := time.Now().Add(15000 * time.Millisecond)
runTestQuery := func() ([]testDoc, error) {
test := map[string]interface{}{
"statement": fmt.Sprintf("SELECT i,testName FROM %s WHERE testName=\"%s\"", nqh.suite.BucketName, nqh.TestName),
"client_context_id": "12345",
}
payload, err := json.Marshal(test)
if err != nil {
nqh.suite.T().Errorf("failed to marshal test payload: %s", err)
}
iterDeadline := time.Now().Add(5000 * time.Millisecond)
if iterDeadline.After(deadline) {
iterDeadline = deadline
}
resCh := make(chan *N1QLRowReader)
errCh := make(chan error)
_, err = ag.N1QLQuery(N1QLQueryOptions{
Payload: payload,
RetryStrategy: nil,
Deadline: iterDeadline,
}, func(reader *N1QLRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
return nil, err
}
var rows *N1QLRowReader
select {
case err := <-errCh:
return nil, err
case res := <-resCh:
rows = res
}
var docs []testDoc
for {
row := rows.NextRow()
if row == nil {
break
}
var doc testDoc
err := json.Unmarshal(row, &doc)
if err != nil {
return nil, err
}
docs = append(docs, doc)
}
err = rows.Err()
if err != nil {
return nil, err
}
return docs, nil
}
lastError := ""
for {
docs, err := runTestQuery()
if err == nil {
testFailed := false
for _, doc := range docs {
if doc.I < 1 || doc.I > nqh.NumDocs {
lastError = fmt.Sprintf("query test read invalid row i=%d", doc.I)
testFailed = true
}
}
numDocs := len(docs)
if numDocs != nqh.NumDocs {
lastError = fmt.Sprintf("query test read invalid number of rows %d!=%d", numDocs, 5)
testFailed = true
}
if !testFailed {
break
}
}
sleepDeadline := time.Now().Add(1000 * time.Millisecond)
if sleepDeadline.After(deadline) {
sleepDeadline = deadline
}
time.Sleep(sleepDeadline.Sub(time.Now()))
if sleepDeadline == deadline {
nqh.suite.T().Errorf("timed out waiting for indexing: %s", lastError)
break
}
}
}
func (nqh *n1qlTestHelper) testN1QLPrepared(t *testing.T) {
ag := nqh.suite.AgentGroup()
deadline := time.Now().Add(15000 * time.Millisecond)
runTestQuery := func() ([]testDoc, error) {
test := map[string]interface{}{
"statement": fmt.Sprintf("SELECT i,testName FROM %s WHERE testName=\"%s\"", nqh.suite.BucketName, nqh.TestName),
"client_context_id": "1234",
}
payload, err := json.Marshal(test)
if err != nil {
nqh.suite.T().Errorf("failed to marshal test payload: %s", err)
}
iterDeadline := time.Now().Add(5000 * time.Millisecond)
if iterDeadline.After(deadline) {
iterDeadline = deadline
}
resCh := make(chan *N1QLRowReader)
errCh := make(chan error)
_, err = ag.PreparedN1QLQuery(N1QLQueryOptions{
Payload: payload,
RetryStrategy: nil,
Deadline: iterDeadline,
}, func(reader *N1QLRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
return nil, err
}
var rows *N1QLRowReader
select {
case err := <-errCh:
return nil, err
case res := <-resCh:
rows = res
}
var docs []testDoc
for {
row := rows.NextRow()
if row == nil {
break
}
var doc testDoc
err := json.Unmarshal(row, &doc)
if err != nil {
return nil, err
}
docs = append(docs, doc)
}
err = rows.Err()
if err != nil {
return nil, err
}
return docs, nil
}
lastError := ""
for {
docs, err := runTestQuery()
if err == nil {
testFailed := false
for _, doc := range docs {
if doc.I < 1 || doc.I > nqh.NumDocs {
lastError = fmt.Sprintf("query test read invalid row i=%d", doc.I)
testFailed = true
}
}
numDocs := len(docs)
if numDocs != nqh.NumDocs {
lastError = fmt.Sprintf("query test read invalid number of rows %d!=%d", numDocs, 5)
testFailed = true
}
if !testFailed {
break
}
}
sleepDeadline := time.Now().Add(1000 * time.Millisecond)
if sleepDeadline.After(deadline) {
sleepDeadline = deadline
}
time.Sleep(sleepDeadline.Sub(time.Now()))
if sleepDeadline == deadline {
nqh.suite.T().Errorf("timed out waiting for indexing: %s", lastError)
break
}
}
}
func (suite *StandardTestSuite) TestN1QL() {
suite.EnsureSupportsFeature(TestFeatureN1ql)
helper := &n1qlTestHelper{
TestName: "testQuery",
NumDocs: 5,
suite: suite,
}
suite.T().Run("setup", helper.testSetupN1ql)
suite.tracer.Reset()
suite.T().Run("Basic", helper.testN1QLBasic)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 1) {
for i := 0; i < len(nilParents); i++ {
suite.AssertHTTPSpan(nilParents[i], "N1QLQuery")
}
}
}
suite.VerifyMetrics(suite.meter, "n1ql:N1QLQuery", 1, true, false)
suite.T().Run("cleanup", helper.testCleanupN1ql)
}
type roundTripper struct {
delay time.Duration
tsport http.RoundTripper
}
func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
<-time.After(rt.delay)
return rt.tsport.RoundTrip(req)
}
func (suite *StandardTestSuite) TestN1QLCancel() {
suite.EnsureSupportsFeature(TestFeatureN1ql)
agent := suite.DefaultAgent()
rt := &roundTripper{delay: 1 * time.Second, tsport: agent.http.cli.Transport}
httpCpt := newHTTPComponentWithClient(
httpComponentProps{},
&http.Client{Transport: rt},
agent.httpMux,
agent.tracer,
)
n1qlCpt := newN1QLQueryComponent(httpCpt, &configManagementComponent{}, &tracerComponent{tracer: suite.tracer, metrics: suite.meter})
resCh := make(chan *N1QLRowReader)
errCh := make(chan error)
payloadStr := `{"statement":"SELECT * FROM test","client_context_id":"12345"}`
op, err := n1qlCpt.N1QLQuery(N1QLQueryOptions{
Payload: []byte(payloadStr),
Deadline: time.Now().Add(5 * time.Second),
}, func(reader *N1QLRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
suite.T().Fatalf("Failed to execute query %s", err)
}
op.Cancel()
var rows *N1QLRowReader
var resErr error
select {
case err := <-errCh:
resErr = err
case res := <-resCh:
rows = res
}
if rows != nil {
suite.T().Fatal("Received rows but should not have")
}
if !errors.Is(resErr, ErrRequestCanceled) {
suite.T().Fatalf("Error should have been request canceled but was %s", resErr)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 1) {
for i := 0; i < len(nilParents); i++ {
suite.AssertHTTPSpan(nilParents[i], "N1QLQuery")
}
}
}
suite.VerifyMetrics(suite.meter, "n1ql:N1QLQuery", 1, true, false)
}
func (suite *StandardTestSuite) TestN1QLTimeout() {
suite.EnsureSupportsFeature(TestFeatureN1ql)
ag := suite.AgentGroup()
resCh := make(chan *N1QLRowReader)
errCh := make(chan error)
payloadStr := fmt.Sprintf(`{"statement":"SELECT * FROM %s LIMIT 1","client_context_id":"12345"}`, suite.BucketName)
_, err := ag.N1QLQuery(N1QLQueryOptions{
Payload: []byte(payloadStr),
Deadline: time.Now().Add(1 * time.Microsecond),
}, func(reader *N1QLRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
suite.T().Fatalf("Failed to execute query %s", err)
}
var rows *N1QLRowReader
var resErr error
select {
case err := <-errCh:
resErr = err
case res := <-resCh:
rows = res
}
if rows != nil {
suite.T().Fatal("Received rows but should not have")
}
if !errors.Is(resErr, ErrTimeout) {
suite.T().Fatalf("Error should have been request canceled but was %s", resErr)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(len(nilParents), 1) {
span := nilParents[0]
suite.Assert().Equal("N1QLQuery", span.Name)
suite.Assert().Equal(1, len(span.Tags))
suite.Assert().Equal("couchbase", span.Tags["db.system"])
suite.Assert().True(span.Finished)
_, ok := span.Spans[spanNameDispatchToServer]
suite.Assert().False(ok)
}
}
suite.VerifyMetrics(suite.meter, "n1ql:N1QLQuery", 1, true, false)
}
func (suite *StandardTestSuite) TestN1QLPrepared() {
suite.EnsureSupportsFeature(TestFeatureN1ql)
helper := &n1qlTestHelper{
TestName: "testPreparedQuery",
NumDocs: 5,
suite: suite,
}
suite.T().Run("setup", helper.testSetupN1ql)
suite.tracer.Reset()
suite.T().Run("Basic", helper.testN1QLPrepared)
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 1) {
for i := 0; i < len(nilParents); i++ {
suite.AssertHTTPSpan(nilParents[i], "PreparedN1QLQuery")
}
}
}
suite.T().Run("cleanup", helper.testCleanupN1ql)
}
func (suite *StandardTestSuite) TestN1QLPreparedCancel() {
suite.EnsureSupportsFeature(TestFeatureN1ql)
agent := suite.DefaultAgent()
rt := &roundTripper{delay: 1 * time.Second, tsport: agent.http.cli.Transport}
httpCpt := newHTTPComponentWithClient(
httpComponentProps{},
&http.Client{Transport: rt},
agent.httpMux,
agent.tracer,
)
n1qlCpt := newN1QLQueryComponent(httpCpt, &configManagementComponent{}, &tracerComponent{tracer: suite.tracer, metrics: suite.meter})
resCh := make(chan *N1QLRowReader)
errCh := make(chan error)
payloadStr := `{"statement":"SELECT * FROM test","client_context_id":"12345"}`
op, err := n1qlCpt.PreparedN1QLQuery(N1QLQueryOptions{
Payload: []byte(payloadStr),
Deadline: time.Now().Add(5 * time.Second),
}, func(reader *N1QLRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
suite.T().Fatalf("Failed to execute query %s", err)
}
op.Cancel()
var rows *N1QLRowReader
var resErr error
select {
case err := <-errCh:
resErr = err
case res := <-resCh:
rows = res
}
if rows != nil {
suite.T().Fatal("Received rows but should not have")
}
if !errors.Is(resErr, ErrRequestCanceled) {
suite.T().Fatalf("Error should have been request canceled but was %s", resErr)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().GreaterOrEqual(len(nilParents), 1) {
for i := 0; i < len(nilParents); i++ {
suite.AssertHTTPSpan(nilParents[i], "PreparedN1QLQuery")
}
}
}
suite.VerifyMetrics(suite.meter, "n1ql:PreparedN1QLQuery", 1, true, false)
}
func (suite *StandardTestSuite) TestN1QLPreparedTimeout() {
suite.EnsureSupportsFeature(TestFeatureN1ql)
ag := suite.AgentGroup()
resCh := make(chan *N1QLRowReader)
errCh := make(chan error)
payloadStr := fmt.Sprintf(`{"statement":"SELECT * FROM %s LIMIT 1","client_context_id":"12345"}`, suite.BucketName)
_, err := ag.PreparedN1QLQuery(N1QLQueryOptions{
Payload: []byte(payloadStr),
Deadline: time.Now().Add(1 * time.Microsecond),
}, func(reader *N1QLRowReader, err error) {
if err != nil {
errCh <- err
return
}
resCh <- reader
})
if err != nil {
suite.T().Fatalf("Failed to execute query %s", err)
}
var rows *N1QLRowReader
var resErr error
select {
case err := <-errCh:
resErr = err
case res := <-resCh:
rows = res
}
if rows != nil {
suite.T().Fatal("Received rows but should not have")
}
if !errors.Is(resErr, ErrTimeout) {
suite.T().Fatalf("Error should have been request canceled but was %s", resErr)
}
if suite.Assert().Contains(suite.tracer.Spans, nil) {
nilParents := suite.tracer.Spans[nil]
if suite.Assert().Equal(len(nilParents), 1) {
span := nilParents[0]
suite.Assert().Equal("PreparedN1QLQuery", span.Name)
suite.Assert().Equal(1, len(span.Tags))
suite.Assert().Equal("couchbase", span.Tags["db.system"])
suite.Assert().True(span.Finished)
_, ok := span.Spans[spanNameDispatchToServer]
suite.Assert().False(ok)
}
}
suite.VerifyMetrics(suite.meter, "n1ql:PreparedN1QLQuery", 1, true, false)
}
func (suite *StandardTestSuite) TestN1QLErrorReasonDocumentExists() {
suite.EnsureSupportsFeature(TestFeatureN1ql)
suite.EnsureSupportsFeature(TestFeatureN1qlReasons)
agent, s := suite.GetAgentAndHarness()
collection := suite.CollectionName
if collection == "" {
collection = "_default"
}
scope := suite.ScopeName
if scope == "" {
scope = "_default"
}
s.PushOp(agent.Set(SetOptions{
Key: []byte("n1qldocumentexists"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set returned error %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
payloadStr := fmt.Sprintf(
`{"statement":"INSERT INTO %s.%s.%s (KEY, VALUE) VALUES (\"n1qldocumentexists\", {\"type\": \"hotel\"})"}`,
suite.BucketName,
scope,
collection,
)
s.PushOp(agent.N1QLQuery(N1QLQueryOptions{
Payload: []byte(payloadStr),
Deadline: time.Now().Add(10 * time.Second),
}, func(reader *N1QLRowReader, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("N1QLQuery operation failed: %v", err)
}
for {
row := reader.NextRow()
if row == nil {
break
}
}
err = reader.Err()
if !errors.Is(err, ErrDocumentExists) {
s.Fatalf("N1QLQuery should failed with document exists, was: %v", err)
}
})
}))
s.Wait(0)
}
// TestN1QLErrorsAndResults tests the case where we receive both errors and results from the server meaning
// that we cannot immediately return an error and must surface it through Err instead.
func (suite *UnitTestSuite) TestN1QLErrorsAndResults() {
d, err := suite.LoadRawTestDataset("query_rows_errors")
suite.Require().Nil(err)
r := ioutil.NopCloser(bytes.NewReader(d))
resp := &HTTPResponse{
Endpoint: "whatever",
StatusCode: 200,
ContentLength: int64(len(d)),
Body: r,
}
configC := new(mockConfigManager)
configC.On("AddConfigWatcher", mock.Anything)
httpC := new(mockHttpComponentInterface)
httpC.On("DoInternalHTTPRequest", mock.AnythingOfType("*gocbcore.httpRequest"), false).
Return(resp, nil)
n1qlC := newN1QLQueryComponent(httpC, configC, newTracerComponent(&noopTracer{}, "", true, &noopMeter{}))
test := map[string]interface{}{
"statement": "SELECT 1=1",
"client_context_id": "1234",
}
payload, err := json.Marshal(test)
suite.Require().Nil(err, err)
waitCh := make(chan *N1QLRowReader)
_, err = n1qlC.N1QLQuery(N1QLQueryOptions{
Payload: payload,
}, func(reader *N1QLRowReader, err error) {
suite.Require().Nil(err, err)
waitCh <- reader
})
suite.Require().Nil(err, err)
reader := <-waitCh
numRows := 0
for reader.NextRow() != nil {
numRows++
}
suite.Assert().Zero(numRows)
err = reader.Err()
suite.Require().NotNil(err)
suite.Assert().True(errors.Is(err, ErrCasMismatch))
var nErr *N1QLError
suite.Require().True(errors.As(err, &nErr))
suite.Require().Len(nErr.Errors, 1)
firstErr := nErr.Errors[0]
suite.Assert().Equal(uint32(12009), firstErr.Code)
suite.Assert().NotEmpty(firstErr.Message)
}
func (suite *UnitTestSuite) TestN1QLOldPreparedErrorsAndResults() {
d, err := suite.LoadRawTestDataset("query_rows_errors")
suite.Require().Nil(err)
r := ioutil.NopCloser(bytes.NewReader(d))
resp := &HTTPResponse{
Endpoint: "whatever",
StatusCode: 200,
ContentLength: int64(len(d)),
Body: r,
}
configC := new(mockConfigManager)
configC.On("AddConfigWatcher", mock.Anything)
httpC := new(mockHttpComponentInterface)
httpC.On("DoInternalHTTPRequest", mock.AnythingOfType("*gocbcore.httpRequest"), false).
Return(resp, nil)
n1qlC := newN1QLQueryComponent(httpC, configC, newTracerComponent(&noopTracer{}, "", true, &noopMeter{}))
test := map[string]interface{}{
"statement": "SELECT 1=1",
"client_context_id": "1234",
}
payload, err := json.Marshal(test)
suite.Require().Nil(err, err)
waitCh := make(chan error)
_, err = n1qlC.PreparedN1QLQuery(N1QLQueryOptions{
Payload: payload,
}, func(reader *N1QLRowReader, err error) {
waitCh <- err
})
suite.Require().Nil(err, err)
err = <-waitCh
var nErr *N1QLError
suite.Require().True(errors.As(err, &nErr))
suite.Require().Len(nErr.Errors, 1)
firstErr := nErr.Errors[0]
suite.Assert().Equal(uint32(12009), firstErr.Code)
suite.Assert().NotEmpty(firstErr.Message)
}
func (suite *UnitTestSuite) TestN1QLOldPreparedUnknownErrorsAndResults() {
d, err := suite.LoadRawTestDataset("query_rows_unknown_errors")
suite.Require().Nil(err)
r := ioutil.NopCloser(bytes.NewReader(d))
resp := &HTTPResponse{
Endpoint: "whatever",
StatusCode: 200,
ContentLength: int64(len(d)),
Body: r,
}
configC := new(mockConfigManager)
configC.On("AddConfigWatcher", mock.Anything)
httpC := new(mockHttpComponentInterface)
httpC.On("DoInternalHTTPRequest", mock.AnythingOfType("*gocbcore.httpRequest"), false).
Return(resp, nil)
n1qlC := newN1QLQueryComponent(httpC, configC, newTracerComponent(&noopTracer{}, "", true, &noopMeter{}))
test := map[string]interface{}{
"statement": "SELECT 1=1",
"client_context_id": "1234",
}
payload, err := json.Marshal(test)
suite.Require().Nil(err, err)
waitCh := make(chan error)
_, err = n1qlC.PreparedN1QLQuery(N1QLQueryOptions{
Payload: payload,
}, func(reader *N1QLRowReader, err error) {
waitCh <- err
})
suite.Require().Nil(err, err)
err = <-waitCh
var nErr *N1QLError
suite.Require().True(errors.As(err, &nErr))
suite.Require().Len(nErr.Errors, 1)
firstErr := nErr.Errors[0]
suite.Assert().Equal(uint32(13014), firstErr.Code)
suite.Assert().NotEmpty(firstErr.Message)
}
func (suite *UnitTestSuite) TestN1QLErrUnknownErrorsAndResults() {
d, err := suite.LoadRawTestDataset("query_rows_unknown_errors")
suite.Require().Nil(err)
r := ioutil.NopCloser(bytes.NewReader(d))
resp := &HTTPResponse{
Endpoint: "whatever",
StatusCode: 200,
ContentLength: int64(len(d)),
Body: r,
}
configC := new(mockConfigManager)
configC.On("AddConfigWatcher", mock.Anything)
httpC := new(mockHttpComponentInterface)
httpC.On("DoInternalHTTPRequest", mock.AnythingOfType("*gocbcore.httpRequest"), false).
Return(resp, nil)
n1qlC := newN1QLQueryComponent(httpC, configC, newTracerComponent(&noopTracer{}, "", true, &noopMeter{}))
test := map[string]interface{}{
"statement": "SELECT 1=1",
"client_context_id": "1234",
}
payload, err := json.Marshal(test)
suite.Require().Nil(err, err)
waitCh := make(chan *N1QLRowReader)
_, err = n1qlC.N1QLQuery(N1QLQueryOptions{
Payload: payload,
}, func(reader *N1QLRowReader, err error) {
suite.Require().Nil(err, err)
waitCh <- reader
})
suite.Require().Nil(err, err)
reader := <-waitCh
numRows := 0
for reader.NextRow() != nil {
numRows++
}
suite.Assert().Zero(numRows)
err = reader.Err()
suite.Require().NotNil(err)
var nErr *N1QLError
suite.Require().True(errors.As(err, &nErr))
suite.Require().Len(nErr.Errors, 1)
firstErr := nErr.Errors[0]
suite.Assert().Equal(uint32(13014), firstErr.Code)
suite.Assert().NotEmpty(firstErr.Message)
}
type readerAndError struct {
reader *N1QLRowReader
err error
}
type n1qlHTTPComponent struct {
Endpoint string
StatusCode int
Body []byte
}
func (nhc *n1qlHTTPComponent) DoInternalHTTPRequest(req *httpRequest, skipConfigCheck bool) (*HTTPResponse, error) {
body := ioutil.NopCloser(bytes.NewReader(nhc.Body))
return &HTTPResponse{
Endpoint: nhc.Endpoint,
StatusCode: nhc.StatusCode,
Body: body,
ContentLength: int64(len(nhc.Body)),
}, nil
}
func (suite *UnitTestSuite) doN1QLRequest(respData []byte, statusCode int, retryStrat RetryStrategy) readerAndError {
configC := new(mockConfigManager)
configC.On("AddConfigWatcher", mock.Anything)
httpC := &n1qlHTTPComponent{
Endpoint: "whatever",
StatusCode: statusCode,
Body: respData,
}
n1qlC := newN1QLQueryComponent(httpC, configC, newTracerComponent(&noopTracer{}, "", true, &noopMeter{}))
test := map[string]interface{}{
"statement": "SELECT 1=1",
"client_context_id": "1234",
}
payload, err := json.Marshal(test)
suite.Require().Nil(err, err)
waitCh := make(chan readerAndError)
_, err = n1qlC.N1QLQuery(N1QLQueryOptions{
Payload: payload,
RetryStrategy: retryStrat,
Deadline: time.Now().Add(1 * time.Second),
}, func(reader *N1QLRowReader, err error) {
waitCh <- readerAndError{reader: reader, err: err}
})
suite.Require().Nil(err, err)
return <-waitCh
}
func (suite *UnitTestSuite) TestN1QLEnhPreparedKnownQueryRetryPrepare4050() {
body := []byte(`{"errors":[{"code":4050,"msg":"Unrecognizable prepared statement"}]}`)
r := ioutil.NopCloser(bytes.NewReader(body))
resp := &HTTPResponse{
Endpoint: "whatever",
StatusCode: 404,
ContentLength: int64(len(body)),
Body: r,
}
body2 := []byte(`{"prepared":"somename","results":[]}`)
r2 := ioutil.NopCloser(bytes.NewReader(body2))
resp2 := &HTTPResponse{
Endpoint: "whatever",
StatusCode: 200,
ContentLength: int64(len(body2)),
Body: r2,
}
configC := new(mockConfigManager)
configC.On("AddConfigWatcher", mock.Anything)
httpC := new(mockHttpComponentInterface)
httpC.On("DoInternalHTTPRequest", mock.AnythingOfType("*gocbcore.httpRequest"), false).
Return(resp, nil).Once().Run(func(args mock.Arguments) {
req := args.Get(0).(*httpRequest)
var body map[string]interface{}
suite.Require().NoError(json.Unmarshal(req.Body, &body))
_, ok := body["statement"]
suite.Assert().False(ok)
prepared := body["prepared"]
suite.Assert().Equal("somename", prepared)
})
httpC.On("DoInternalHTTPRequest", mock.AnythingOfType("*gocbcore.httpRequest"), false).
Return(resp2, nil).Once().Run(func(args mock.Arguments) {
req := args.Get(0).(*httpRequest)
var body map[string]interface{}
suite.Require().NoError(json.Unmarshal(req.Body, &body))
statement := body["statement"]
suite.Assert().Equal("PREPARE SELECT 1=1", statement)
autoExec := body["auto_execute"]
suite.Assert().True(autoExec.(bool))
})
n1qlC := newN1QLQueryComponent(httpC, configC, newTracerComponent(&noopTracer{}, "", true, &noopMeter{}))
n1qlC.enhancedPreparedSupported = 1
n1qlC.queryCache.Put("SELECT 1=1", &n1qlQueryCacheEntry{
name: "somename",
})
test := map[string]interface{}{
"statement": "SELECT 1=1",
"client_context_id": "1234",
}
payload, err := json.Marshal(test)
suite.Require().Nil(err, err)
waitCh := make(chan error, 1)
_, err = n1qlC.PreparedN1QLQuery(N1QLQueryOptions{
Payload: payload,
RetryStrategy: NewBestEffortRetryStrategy(nil),
Deadline: time.Now().Add(1 * time.Second),
}, func(reader *N1QLRowReader, err error) {
waitCh <- err
})
suite.Require().NoError(err, err)
suite.Require().NoError(<-waitCh)
}
func (suite *UnitTestSuite) TestN1QLEnhPreparedKnownQueryFailReprepare() {
body := []byte(`{"errors":[{"code":4050,"msg":"Unrecognizable prepared statement"}]}`)
r := ioutil.NopCloser(bytes.NewReader(body))
resp := &HTTPResponse{
Endpoint: "whatever",
StatusCode: 404,
ContentLength: int64(len(body)),
Body: r,
}
body2 := []byte(`{"errors":[{"code":9999,"msg":"A made up error"}]}`)
r2 := ioutil.NopCloser(bytes.NewReader(body2))
resp2 := &HTTPResponse{
Endpoint: "whatever",
StatusCode: 404,
ContentLength: int64(len(body2)),
Body: r2,
}
configC := new(mockConfigManager)
configC.On("AddConfigWatcher", mock.Anything)
httpC := new(mockHttpComponentInterface)
httpC.On("DoInternalHTTPRequest", mock.AnythingOfType("*gocbcore.httpRequest"), false).
Return(resp, nil).Once()
httpC.On("DoInternalHTTPRequest", mock.AnythingOfType("*gocbcore.httpRequest"), false).
Return(resp2, nil).Once()
n1qlC := newN1QLQueryComponent(httpC, configC, newTracerComponent(&noopTracer{}, "", true, &noopMeter{}))
n1qlC.enhancedPreparedSupported = 1
n1qlC.queryCache.Put("SELECT 1=1", &n1qlQueryCacheEntry{
name: "somename",
})
test := map[string]interface{}{
"statement": "SELECT 1=1",
"client_context_id": "1234",
}
payload, err := json.Marshal(test)
suite.Require().Nil(err, err)
waitCh := make(chan error, 1)
_, err = n1qlC.PreparedN1QLQuery(N1QLQueryOptions{
Payload: payload,
RetryStrategy: NewBestEffortRetryStrategy(nil),
Deadline: time.Now().Add(100 * time.Millisecond),
}, func(reader *N1QLRowReader, err error) {
waitCh <- err
})
suite.Require().NoError(err, err)
var n1qlErr *N1QLError
suite.Require().ErrorAs(<-waitCh, &n1qlErr)
suite.Assert().Equal(uint32(1), n1qlErr.RetryAttempts)
suite.Assert().Contains(n1qlErr.RetryReasons, QueryPreparedStatementFailureRetryReason)
}
type n1qlRetryStrategy struct {
maxAttempts uint32
retries int
}
func (mrs *n1qlRetryStrategy) RetryAfter(req RetryRequest, reason RetryReason) RetryAction {
if req.RetryAttempts() >= mrs.maxAttempts {
return &NoRetryRetryAction{}
}
mrs.retries++
return &WithDurationRetryAction{WithDuration: 1 * time.Millisecond}
}
func (suite *UnitTestSuite) TestN1QLRetryTrueErrorReadOnly() {
d, err := suite.LoadRawTestDataset("query_failure_retry_true")
suite.Require().Nil(err)
mrs := &n1qlRetryStrategy{maxAttempts: 3}
reader := suite.doN1QLRequest(d, 500, mrs)
suite.Assert().Nil(reader.reader)
err = reader.err
suite.Require().NotNil(err)
var nErr *N1QLError
suite.Require().True(errors.As(err, &nErr))
suite.Require().Len(nErr.Errors, 1)
firstErr := nErr.Errors[0]
suite.Assert().Equal(uint32(99999), firstErr.Code)
suite.Assert().Equal("some nonsense", firstErr.Message)
suite.Assert().True(firstErr.Retry)
suite.Assert().NotNil(firstErr.Reason)
suite.Assert().Equal(3, mrs.retries)
}
func (suite *UnitTestSuite) TestN1QLCasMismatch() {
d, err := suite.LoadRawTestDataset("query_failure_cas_mismatch_71")
suite.Require().Nil(err)
result := suite.doN1QLRequest(d, 200, nil)
suite.Require().Nil(result.err, result.err)
reader := result.reader
numRows := 0
for reader.NextRow() != nil {
numRows++
}
suite.Assert().Zero(numRows)
err = reader.Err()
suite.Require().NotNil(err)
var nErr *N1QLError
suite.Require().True(errors.As(err, &nErr))
suite.Require().Len(nErr.Errors, 1)
firstErr := nErr.Errors[0]
suite.Assert().Equal(uint32(12009), firstErr.Code)
suite.Assert().Equal("some other message not matching on cas...", firstErr.Message)
suite.Assert().False(firstErr.Retry)
suite.Assert().NotNil(firstErr.Reason)
suite.Assert().True(errors.Is(err, ErrCasMismatch), "Expected doc not found but was %s", err)
}
func (suite *UnitTestSuite) TestN1QLDocExists() {
d, err := suite.LoadRawTestDataset("query_failure_doc_exists_71")
suite.Require().Nil(err)
result := suite.doN1QLRequest(d, 200, nil)
suite.Require().Nil(result.err, result.err)
reader := result.reader
numRows := 0
for reader.NextRow() != nil {
numRows++
}
suite.Assert().Zero(numRows)
err = reader.Err()
suite.Require().NotNil(err)
var nErr *N1QLError
suite.Require().True(errors.As(err, &nErr))
suite.Require().Len(nErr.Errors, 1)
firstErr := nErr.Errors[0]
suite.Assert().Equal(uint32(12009), firstErr.Code)
suite.Assert().Equal("some message", firstErr.Message)
suite.Assert().False(firstErr.Retry)
suite.Assert().NotNil(firstErr.Reason)
suite.Assert().True(errors.Is(err, ErrDocumentExists), "Expected doc not found but was %s", err)
}
func (suite *UnitTestSuite) TestN1QLDocNotFound() {
d, err := suite.LoadRawTestDataset("query_failure_doc_not_found_71")
suite.Require().Nil(err)
result := suite.doN1QLRequest(d, 200, nil)
suite.Require().Nil(result.err, result.err)
reader := result.reader
numRows := 0
for reader.NextRow() != nil {
numRows++
}
suite.Assert().Zero(numRows)
err = reader.Err()
suite.Require().NotNil(err)
var nErr *N1QLError
suite.Require().True(errors.As(err, &nErr))
suite.Require().Len(nErr.Errors, 1)
firstErr := nErr.Errors[0]
suite.Assert().Equal(uint32(12009), firstErr.Code)
suite.Assert().Equal("some message", firstErr.Message)
suite.Assert().False(firstErr.Retry)
suite.Assert().NotNil(firstErr.Reason)
suite.Assert().True(errors.Is(err, ErrDocumentNotFound), "Expected doc not found but was %s", err)
}
type n1qlBodyErrDesc struct {
Code uint32
Message string `json:"msg"`
Retry bool
Reason map[string]interface{}
}
type n1qlBody struct {
RequestID string `json:"requestID"`
ClientContextID string `json:"clientContextID"`
Signature map[string]string `json:"signature"`
Results []interface{} `json:"results"`
Errors []n1qlBodyErrDesc `json:"errors"`
Status string `json:"status"`
Metrics struct {
ElapsedTime int `json:"elapsedTime"`
ExecutionTime int `json:"executionTime"`
ResultCount int `json:"resultCount"`
ResultSize int `json:"resultSize"`
ErrorCount int `json:"errorCount"`
} `json:"metrics"`
}
func (suite *UnitTestSuite) TestN1QLMB50643() {
body := n1qlBody{
RequestID: "1234",
ClientContextID: "12345",
Signature: map[string]string{"*": "*"},
Results: []interface{}{},
Errors: []n1qlBodyErrDesc{
{
Code: 12016,
Message: "MB50643",
Retry: true,
Reason: map[string]interface{}{
"name": "#primary",
},
},
},
Status: "errors",
Metrics: struct {
ElapsedTime int `json:"elapsedTime"`
ExecutionTime int `json:"executionTime"`
ResultCount int `json:"resultCount"`
ResultSize int `json:"resultSize"`
ErrorCount int `json:"errorCount"`
}{
ErrorCount: 1,
},
}
d, err := json.Marshal(body)
suite.Require().Nil(err)
mrs := &n1qlRetryStrategy{maxAttempts: 3}
reader := suite.doN1QLRequest(d, 500, mrs)
suite.Assert().Nil(reader.reader)
err = reader.err
suite.Require().NotNil(err)
var nErr *N1QLError
suite.Require().True(errors.As(err, &nErr))
suite.Require().Len(nErr.Errors, 1)
firstErr := nErr.Errors[0]
suite.Assert().Equal(uint32(12016), firstErr.Code)
suite.Assert().Equal("MB50643", firstErr.Message)
suite.Assert().True(firstErr.Retry)
suite.Assert().NotNil(firstErr.Reason)
suite.Assert().Equal(0, mrs.retries)
suite.Assert().True(errors.Is(err, ErrIndexFailure), "Expected doc not found but was %s", err)
}
gocbcore-10.2.3/nodeversion_test.go 0000664 0000000 0000000 00000006171 14417540156 0017315 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"fmt"
"strconv"
"strings"
)
type NodeVersion struct {
Major int
Minor int
Patch int
Build int
Edition NodeEdition
Modifier string
}
type NodeEdition int
const (
CommunityNodeEdition = NodeEdition(1)
EnterpriseNodeEdition = NodeEdition(2)
)
func (v NodeVersion) Equal(ov NodeVersion) bool {
if v.Major == ov.Major && v.Minor == ov.Minor &&
v.Patch == ov.Patch && v.Build == ov.Build && v.Edition == ov.Edition && v.Modifier == ov.Modifier {
return true
}
return false
}
func (v NodeVersion) Higher(ov NodeVersion) bool {
if v.Major > ov.Major {
return true
} else if v.Major < ov.Major {
return false
}
if v.Minor > ov.Minor {
return true
} else if v.Minor < ov.Minor {
return false
}
if v.Patch > ov.Patch {
return true
} else if v.Patch < ov.Patch {
return false
}
if v.Build > ov.Build {
return true
} else if v.Build < ov.Build {
return false
}
if v.Edition > ov.Edition {
return true
}
return false
}
func (v NodeVersion) Lower(ov NodeVersion) bool {
return !v.Higher(ov) && !v.Equal(ov)
}
func nodeVersionFromString(version string) (*NodeVersion, error) {
vSplit := strings.Split(version, ".")
lenSplit := len(vSplit)
if lenSplit == 0 {
return nil, fmt.Errorf("must provide at least a major version")
}
var err error
nodeVersion := NodeVersion{}
nodeVersion.Major, err = strconv.Atoi(vSplit[0])
if err != nil {
return nil, fmt.Errorf("major version is not a valid integer")
}
if lenSplit == 1 {
return &nodeVersion, nil
}
nodeVersion.Minor, err = strconv.Atoi(vSplit[1])
if err != nil {
return nil, fmt.Errorf("minor version is not a valid integer")
}
if lenSplit == 2 {
return &nodeVersion, nil
}
nodeBuild := strings.Split(vSplit[2], "-")
nodeVersion.Patch, err = strconv.Atoi(nodeBuild[0])
if err != nil {
return nil, fmt.Errorf("patch version is not a valid integer")
}
if len(nodeBuild) == 1 {
return &nodeVersion, nil
}
buildEdition := strings.Split(nodeBuild[1], "-")
nodeVersion.Build, err = strconv.Atoi(buildEdition[0])
if err != nil {
edition, modifier, err := editionModifierFromString(buildEdition[0])
if err != nil {
return nil, err
}
nodeVersion.Edition = edition
nodeVersion.Modifier = modifier
return &nodeVersion, nil
}
if len(buildEdition) == 1 {
return &nodeVersion, nil
}
edition, modifier, err := editionModifierFromString(buildEdition[1])
if err != nil {
return nil, err
}
nodeVersion.Edition = edition
nodeVersion.Modifier = modifier
return &nodeVersion, nil
}
func editionModifierFromString(editionModifier string) (NodeEdition, string, error) {
split := strings.Split(editionModifier, "-")
editionStr := strings.ToLower(split[0])
var edition NodeEdition
var modifier string
if editionStr == "enterprise" {
edition = EnterpriseNodeEdition
} else if editionStr == "community" {
edition = CommunityNodeEdition
} else if editionStr == "dp" {
modifier = editionStr
} else {
return 0, "", errors.New("Unrecognised edition or modifier: " + editionStr)
}
if len(split) == 1 {
return edition, modifier, nil
}
return edition, strings.ToLower(split[1]), nil
}
gocbcore-10.2.3/ns_server_test.go 0000664 0000000 0000000 00000026452 14417540156 0016774 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"crypto/x509"
"errors"
"strings"
"time"
)
func (suite *StandardTestSuite) VerifyNSKVListTLS(seed string, endpoints []routeEndpoint) {
for _, ep := range endpoints {
epParts := strings.Split(ep.Address, "://")
hostport := strings.Split(epParts[1], ":")
if hostport[0] == seed {
suite.Assert().Equal("couchbase", epParts[0])
suite.Assert().Equal("11210", hostport[1])
suite.Assert().True(ep.IsSeedNode)
} else {
suite.Assert().Equal("couchbases", epParts[0])
suite.Assert().Equal("11207", hostport[1])
suite.Assert().False(ep.IsSeedNode)
}
}
}
func (suite *StandardTestSuite) VerifyNSKVListNonTLS(endpoints []routeEndpoint) {
for _, ep := range endpoints {
epParts := strings.Split(ep.Address, "://")
hostport := strings.Split(epParts[1], ":")
suite.Assert().Equal("couchbase", epParts[0])
suite.Assert().Equal("11210", hostport[1])
}
}
func (suite *StandardTestSuite) VerifyNSMgmtListTLS(seed string, endpoints []routeEndpoint) {
for _, ep := range endpoints {
epParts := strings.Split(ep.Address, "://")
hostport := strings.Split(epParts[1], ":")
if hostport[0] == seed {
suite.Assert().Equal("http", epParts[0])
suite.Assert().Equal("8091", hostport[1])
suite.Assert().True(ep.IsSeedNode)
} else {
suite.Assert().Equal("https", epParts[0])
suite.Assert().Equal("18091", hostport[1])
suite.Assert().False(ep.IsSeedNode)
}
}
}
func (suite *StandardTestSuite) VerifyNSMgmtListNonTLS(endpoints []routeEndpoint) {
for _, ep := range endpoints {
epParts := strings.Split(ep.Address, "://")
hostport := strings.Split(epParts[1], ":")
suite.Assert().Equal("http", epParts[0])
suite.Assert().Equal("8091", hostport[1])
}
}
func (suite *StandardTestSuite) TestReconfigureSecurity() {
suite.EnsureSupportsFeature(TestFeatureSsl)
// This will create a config with TLS enabled
config, seedAddr := suite.CreateNSAgentConfig()
splitSeed := strings.Split(seedAddr, ":")
agent, err := CreateAgent(config)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "test", suite.CollectionName, suite.ScopeName)
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurity")
err = agent.ReconfigureSecurity(ReconfigureSecurityOptions{
UseTLS: false,
})
suite.Require().Nil(err, err)
success := suite.tryUntil(time.Now().Add(5*time.Second), 100*time.Millisecond, func() bool {
kvMuxState := agent.kvMux.getState()
kvEps := kvMuxState.kvServerList
for _, ep := range kvEps {
epParts := strings.Split(ep.Address, "://")
hostport := strings.Split(epParts[1], ":")
if hostport[1] == "11207" {
suite.T().Logf("Encrypted endpoint found: %s", ep.Address)
return false
}
}
return true
})
suite.Require().True(success, "One or more endpoints never switched to nonTLS")
kvMuxState := agent.kvMux.getState()
httpMuxState := agent.httpMux.Get()
suite.VerifyNSKVListNonTLS(kvMuxState.kvServerList)
suite.VerifyNSMgmtListNonTLS(httpMuxState.mgmtEpList)
suite.VerifyConnectedToBucket(agent, s, "TestReconfigureSecurity", suite.CollectionName, suite.ScopeName)
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurity")
err = agent.ReconfigureSecurity(ReconfigureSecurityOptions{
UseTLS: true,
TLSRootCAProvider: func() *x509.CertPool {
return nil
},
})
suite.Require().Nil(err, err)
success = suite.tryUntil(time.Now().Add(5*time.Second), 100*time.Millisecond, func() bool {
kvMuxState := agent.kvMux.getState()
kvEps := kvMuxState.kvServerList
for _, ep := range kvEps {
epParts := strings.Split(ep.Address, "://")
hostport := strings.Split(epParts[1], ":")
if hostport[0] != splitSeed[0] && hostport[1] == "8091" {
suite.T().Logf("Unencrypted endpoint found: %s", ep.Address)
return false
}
}
return true
})
suite.Require().True(success, "One or more endpoints never switched to TLS")
kvMuxState = agent.kvMux.getState()
httpMuxState = agent.httpMux.Get()
suite.VerifyNSKVListTLS(splitSeed[0], kvMuxState.kvServerList)
suite.VerifyNSMgmtListTLS(splitSeed[0], httpMuxState.mgmtEpList)
suite.VerifyConnectedToBucket(agent, s, "TestReconfigureSecurity", suite.CollectionName, suite.ScopeName)
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurity")
}
func (suite *StandardTestSuite) TestReconfigureSecurityChangeAuthMechanisms() {
suite.EnsureSupportsFeature(TestFeatureSsl)
globalTestLogger.SuppressWarnings(true)
defer globalTestLogger.SuppressWarnings(false)
// This will create a config with TLS enabled
config, _ := suite.CreateNSAgentConfig()
agent, err := CreateAgent(config)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestReconfigureSecurityChangeAuthProvider", suite.CollectionName, suite.ScopeName)
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurityChangeAuthProvider")
err = agent.ReconfigureSecurity(ReconfigureSecurityOptions{
AuthMechanisms: []AuthMechanism{PlainAuthMechanism},
})
suite.Require().Nil(err, err)
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitUntilReady failed with error: %v", err)
}
})
}))
s.Wait(6)
suite.VerifyConnectedToBucket(agent, s, "TestReconfigureSecurityChangeAuthMechanisms", suite.CollectionName, suite.ScopeName)
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurityChangeAuthProvider")
}
func (suite *StandardTestSuite) TestReconfigureSecurityChangeAuthProvider() {
suite.EnsureSupportsFeature(TestFeatureSsl)
globalTestLogger.SuppressWarnings(true)
defer globalTestLogger.SuppressWarnings(false)
// This will create a config with TLS enabled
config, _ := suite.CreateNSAgentConfig()
// Reduce this otherwise our forced auth errors are going to cause bootstrap backoff of 5s.
config.KVConfig.ServerWaitBackoff = 500 * time.Millisecond
agent, err := CreateAgent(config)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestReconfigureSecurityChangeAuthProvider", suite.CollectionName, suite.ScopeName)
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurityChangeAuthProvider")
err = agent.ReconfigureSecurity(ReconfigureSecurityOptions{
UseTLS: false,
Auth: PasswordAuthProvider{
Username: "",
Password: "",
},
})
suite.Require().Nil(err, err)
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if !errors.Is(err, ErrAuthenticationFailure) {
s.Fatalf("WaitUntilReady should have failed with auth error but was: %v", err)
}
})
}))
s.Wait(6)
err = agent.ReconfigureSecurity(ReconfigureSecurityOptions{
UseTLS: false,
Auth: globalTestConfig.Authenticator,
})
suite.Require().Nil(err, err)
suite.VerifyConnectedToBucket(agent, s, "TestReconfigureSecurityChangeAuthProvider", suite.CollectionName, suite.ScopeName)
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurityChangeAuthProvider")
err = agent.ReconfigureSecurity(ReconfigureSecurityOptions{
UseTLS: true,
TLSRootCAProvider: func() *x509.CertPool {
return nil
},
Auth: globalTestConfig.Authenticator,
})
suite.Require().Nil(err, err)
suite.VerifyConnectedToBucket(agent, s, "TestReconfigureSecurityChangeAuthProvider", suite.CollectionName, suite.ScopeName)
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurityChangeAuthProvider")
}
func (suite *StandardTestSuite) TestReconfigureSecurityNotNSServer() {
agent := suite.DefaultAgent()
err := agent.ReconfigureSecurity(ReconfigureSecurityOptions{
UseTLS: false,
})
suite.Require().NotNil(err, err)
}
func (suite *StandardTestSuite) TestReconfigureSecurityTLSNoProvider() {
suite.EnsureSupportsFeature(TestFeatureSsl)
globalTestLogger.SuppressWarnings(true)
defer globalTestLogger.SuppressWarnings(false)
// This will create a config with TLS enabled
config, _ := suite.CreateNSAgentConfig()
// Reduce this otherwise our forced auth errors are going to cause bootstrap backoff of 5s.
config.KVConfig.ServerWaitBackoff = 500 * time.Millisecond
agent, err := CreateAgent(config)
suite.Require().Nil(err, err)
defer agent.Close()
err = agent.ReconfigureSecurity(ReconfigureSecurityOptions{
UseTLS: true,
})
suite.Require().NotNil(err, err)
}
func (suite *StandardTestSuite) TestReconfigureSecurityMemd() {
suite.EnsureSupportsFeature(TestFeatureSsl)
suite.EnsureSupportsFeature(TestFeatureMemd)
// This will create a config with TLS enabled
config, seedAddr := suite.CreateNSAgentConfig()
splitSeed := strings.Split(seedAddr, ":")
config.BucketName = globalTestConfig.MemdBucketName
agent, err := CreateAgent(config)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestReconfigureSecurityMemd", "", "")
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurityMemd")
err = agent.ReconfigureSecurity(ReconfigureSecurityOptions{
UseTLS: false,
})
suite.Require().Nil(err, err)
success := suite.tryUntil(time.Now().Add(5*time.Second), 100*time.Millisecond, func() bool {
kvMuxState := agent.kvMux.getState()
kvEps := kvMuxState.kvServerList
for _, ep := range kvEps {
epParts := strings.Split(ep.Address, "://")
hostport := strings.Split(epParts[1], ":")
if hostport[1] == "11207" {
suite.T().Logf("Encrypted endpoint found: %s", ep.Address)
return false
}
}
return true
})
suite.Require().True(success, "One or more endpoints never switched to nonTLS")
kvMuxState := agent.kvMux.getState()
httpMuxState := agent.httpMux.Get()
suite.VerifyNSKVListNonTLS(kvMuxState.kvServerList)
suite.VerifyNSMgmtListNonTLS(httpMuxState.mgmtEpList)
suite.VerifyConnectedToBucket(agent, s, "TestReconfigureSecurityMemd", "", "")
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurityMemd")
err = agent.ReconfigureSecurity(ReconfigureSecurityOptions{
UseTLS: true,
TLSRootCAProvider: func() *x509.CertPool {
return nil
},
})
suite.Require().Nil(err, err)
success = suite.tryUntil(time.Now().Add(5*time.Second), 100*time.Millisecond, func() bool {
kvMuxState := agent.kvMux.getState()
kvEps := kvMuxState.kvServerList
for _, ep := range kvEps {
epParts := strings.Split(ep.Address, "://")
hostport := strings.Split(epParts[1], ":")
if hostport[0] != splitSeed[0] && hostport[1] == "8091" {
suite.T().Logf("Unencrypted endpoint found: %s", ep.Address)
return false
}
}
return true
})
suite.Require().True(success, "One or more endpoints never switched to TLS")
kvMuxState = agent.kvMux.getState()
httpMuxState = agent.httpMux.Get()
suite.VerifyNSKVListTLS(splitSeed[0], kvMuxState.kvServerList)
suite.VerifyNSMgmtListTLS(splitSeed[0], httpMuxState.mgmtEpList)
suite.VerifyConnectedToBucket(agent, s, "TestReconfigureSecurityMemd", "", "")
suite.VerifyConnectedToBucketHTTP(agent, globalTestConfig.BucketName, s, "TestReconfigureSecurityMemd")
}
gocbcore-10.2.3/observecomponent.go 0000664 0000000 0000000 00000016371 14417540156 0017316 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/binary"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
type bucketUtilsProvider interface {
KeyToVbucket(key []byte) (uint16, error)
BucketType() bucketType
}
type observeComponent struct {
cidMgr *collectionsComponent
defaultRetryStrategy RetryStrategy
tracer *tracerComponent
bucketUtils bucketUtilsProvider
}
func newObserveComponent(cidMgr *collectionsComponent, defaultRetryStrategy RetryStrategy, tracerCmpt *tracerComponent,
bucketUtils bucketUtilsProvider) *observeComponent {
return &observeComponent{
cidMgr: cidMgr,
defaultRetryStrategy: defaultRetryStrategy,
tracer: tracerCmpt,
bucketUtils: bucketUtils,
}
}
func (oc *observeComponent) Observe(opts ObserveOptions, cb ObserveCallback) (PendingOp, error) {
tracer := oc.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "Observe", opts.TraceContext)
if oc.bucketUtils.BucketType() != bktTypeCouchbase {
tracer.Finish()
return nil, errFeatureNotAvailable
}
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
if len(resp.Value) < 4 {
tracer.Finish()
cb(nil, errProtocol)
return
}
keyLen := int(binary.BigEndian.Uint16(resp.Value[2:]))
if len(resp.Value) != 2+2+keyLen+1+8 {
tracer.Finish()
cb(nil, errProtocol)
return
}
keyState := memd.KeyState(resp.Value[2+2+keyLen])
cas := binary.BigEndian.Uint64(resp.Value[2+2+keyLen+1:])
res := &ObserveResult{
KeyState: keyState,
Cas: Cas(cas),
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
vbID, err := oc.bucketUtils.KeyToVbucket(opts.Key)
if err != nil {
tracer.Finish()
return nil, err
}
keyLen := len(opts.Key)
valueBuf := make([]byte, 2+2+keyLen)
binary.BigEndian.PutUint16(valueBuf[0:], vbID)
binary.BigEndian.PutUint16(valueBuf[2:], uint16(keyLen))
copy(valueBuf[4:], opts.Key)
if opts.RetryStrategy == nil {
opts.RetryStrategy = oc.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdObserve,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: nil,
Value: valueBuf,
Vbucket: vbID,
CollectionID: opts.CollectionID,
UserImpersonationFrame: userFrame,
},
ReplicaIdx: opts.ReplicaIdx,
Callback: handler,
RootTraceContext: tracer.RootContext(),
CollectionName: opts.CollectionName,
ScopeName: opts.ScopeName,
RetryStrategy: opts.RetryStrategy,
}
op, err := oc.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallbackAndFinishTracer(&TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "Unlock",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
}, tracer)
}))
}
return op, nil
}
func (oc *observeComponent) ObserveVb(opts ObserveVbOptions, cb ObserveVbCallback) (PendingOp, error) {
tracer := oc.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "ObserveVb", opts.TraceContext)
if oc.bucketUtils.BucketType() != bktTypeCouchbase {
tracer.Finish()
return nil, errFeatureNotAvailable
}
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
if err != nil {
tracer.Finish()
cb(nil, err)
return
}
if len(resp.Value) < 1 {
tracer.Finish()
cb(nil, errProtocol)
return
}
formatType := resp.Value[0]
if formatType == 0 {
// Normal
if len(resp.Value) < 27 {
tracer.Finish()
cb(nil, errProtocol)
return
}
vbID := binary.BigEndian.Uint16(resp.Value[1:])
vbUUID := binary.BigEndian.Uint64(resp.Value[3:])
persistSeqNo := binary.BigEndian.Uint64(resp.Value[11:])
currentSeqNo := binary.BigEndian.Uint64(resp.Value[19:])
res := &ObserveVbResult{
DidFailover: false,
VbID: vbID,
VbUUID: VbUUID(vbUUID),
PersistSeqNo: SeqNo(persistSeqNo),
CurrentSeqNo: SeqNo(currentSeqNo),
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
return
} else if formatType == 1 {
// Hard Failover
if len(resp.Value) < 43 {
cb(nil, errProtocol)
return
}
vbID := binary.BigEndian.Uint16(resp.Value[1:])
vbUUID := binary.BigEndian.Uint64(resp.Value[3:])
persistSeqNo := binary.BigEndian.Uint64(resp.Value[11:])
currentSeqNo := binary.BigEndian.Uint64(resp.Value[19:])
oldVbUUID := binary.BigEndian.Uint64(resp.Value[27:])
lastSeqNo := binary.BigEndian.Uint64(resp.Value[35:])
res := &ObserveVbResult{
DidFailover: true,
VbID: vbID,
VbUUID: VbUUID(vbUUID),
PersistSeqNo: SeqNo(persistSeqNo),
CurrentSeqNo: SeqNo(currentSeqNo),
OldVbUUID: VbUUID(oldVbUUID),
LastSeqNo: SeqNo(lastSeqNo),
}
res.Internal.ResourceUnits = req.ResourceUnits()
tracer.Finish()
cb(res, nil)
return
} else {
tracer.Finish()
cb(nil, errProtocol)
return
}
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
valueBuf := make([]byte, 8)
binary.BigEndian.PutUint64(valueBuf[0:], uint64(opts.VbUUID))
if opts.RetryStrategy == nil {
opts.RetryStrategy = oc.defaultRetryStrategy
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdObserveSeqNo,
Datatype: 0,
Cas: 0,
Extras: nil,
Key: nil,
Value: valueBuf,
Vbucket: opts.VbID,
UserImpersonationFrame: userFrame,
},
ReplicaIdx: opts.ReplicaIdx,
Callback: handler,
RootTraceContext: tracer.RootContext(),
RetryStrategy: opts.RetryStrategy,
}
op, err := oc.cidMgr.Dispatch(req)
if err != nil {
tracer.Finish()
return nil, err
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallbackAndFinishTracer(&TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "Unlock",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
}, tracer)
}))
}
return op, nil
}
gocbcore-10.2.3/pendingop.go 0000664 0000000 0000000 00000001314 14417540156 0015700 0 ustar 00root root 0000000 0000000 package gocbcore
import "sync/atomic"
// PendingOp represents an outstanding operation within the client.
// This can be used to cancel an operation before it completes.
// This can also be used to Get information about the operation once
// it has completed (cancelled or successful).
type PendingOp interface {
Cancel()
}
type multiPendingOp struct {
ops []PendingOp
completedOps uint32
isIdempotent bool
}
func (mp *multiPendingOp) Cancel() {
for _, op := range mp.ops {
op.Cancel()
}
}
func (mp *multiPendingOp) CompletedOps() uint32 {
return atomic.LoadUint32(&mp.completedOps)
}
func (mp *multiPendingOp) IncrementCompletedOps() uint32 {
return atomic.AddUint32(&mp.completedOps, 1)
}
gocbcore-10.2.3/pipelinesnapshot.go 0000664 0000000 0000000 00000001453 14417540156 0017306 0 ustar 00root root 0000000 0000000 package gocbcore
type pipelineSnapshot struct {
state *kvMuxState
idx int
}
func (pi pipelineSnapshot) RevID() int64 {
return pi.state.RevID()
}
func (pi pipelineSnapshot) NumPipelines() int {
return pi.state.NumPipelines()
}
func (pi pipelineSnapshot) PipelineAt(idx int) *memdPipeline {
return pi.state.GetPipeline(idx)
}
func (pi pipelineSnapshot) Iterate(offset int, cb func(*memdPipeline) bool) {
l := pi.state.NumPipelines()
pi.idx = offset
for iters := 0; iters < l; iters++ {
pi.idx = (pi.idx + 1) % l
p := pi.state.GetPipeline(pi.idx)
if cb(p) {
return
}
}
}
func (pi pipelineSnapshot) NodeByVbucket(vbID uint16, replicaID uint32) (int, error) {
if pi.state.VBMap() == nil {
return 0, errUnsupportedOperation
}
return pi.state.VBMap().NodeByVbucket(vbID, replicaID)
}
gocbcore-10.2.3/pollercontroller.go 0000664 0000000 0000000 00000012373 14417540156 0017325 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
"sync"
"sync/atomic"
)
type pollerController struct {
activeController configPoller
controllerLock sync.Mutex
stopped bool
bucketConfigSeen uint32
cccpPoller *cccpConfigController
httpPoller *httpConfigController
cfgMgr configManager
isFallbackErrorFn func(error) bool
}
type configPollerController interface {
Run()
Stop()
Done() chan struct{}
PollerError() error
ForceHTTPPoller()
}
type configPoller interface {
Done() chan struct{}
Stop()
Reset()
Error() error
}
func newPollerController(cccpPoller *cccpConfigController, httpPoller *httpConfigController, cfgMgr configManager,
errorFn func(error) bool) *pollerController {
pc := &pollerController{
cccpPoller: cccpPoller,
httpPoller: httpPoller,
cfgMgr: cfgMgr,
isFallbackErrorFn: errorFn,
}
cfgMgr.AddConfigWatcher(pc)
return pc
}
// OnNewRouteConfig listens out for every config that comes in so that we (re)start the cccp if applicable.
func (pc *pollerController) OnNewRouteConfig(cfg *routeConfig) {
if cfg.bktType != bktTypeCouchbase && cfg.bktType != bktTypeMemcached {
return
}
atomic.SwapUint32(&pc.bucketConfigSeen, 1)
if cfg.bktType == bktTypeMemcached {
return
}
go func() {
pc.controllerLock.Lock()
if pc.stopped {
pc.controllerLock.Unlock()
return
}
if pc.activeController == pc.httpPoller {
logInfof("Found couchbase bucket and HTTP poller in use. Restarting poller run loop to start cccp.")
pc.activeController = nil
pc.controllerLock.Unlock()
// Stopping the poller will trigger the run loop to loop again.
pc.httpPoller.Stop()
return
}
pc.controllerLock.Unlock()
}()
}
func (pc *pollerController) Run() {
for {
logDebugf("Starting poller controller loop")
pc.controllerLock.Lock()
if pc.stopped {
pc.controllerLock.Unlock()
logDebugf("Poller controller stopped, exiting")
return
}
if pc.httpPoller != nil {
pc.httpPoller.Reset()
}
pc.cccpPoller.Reset()
atomic.SwapUint32(&pc.bucketConfigSeen, 0)
pc.activeController = pc.cccpPoller
pc.controllerLock.Unlock()
err := pc.cccpPoller.DoLoop()
if err != nil {
logDebugf("CCCP poller has exited with err: %v", err)
}
if atomic.LoadUint32(&pc.bucketConfigSeen) == 1 {
logInfof("Config seen but CCCP poller exited, restarting CCCP poller.")
// CCCP managed to fetch a config whilst we were waiting for shutdown, in this case we want to just
// start CCCP again as the bucket must exist and be a couchbase bucket.
continue
}
pc.controllerLock.Lock()
if pc.stopped {
pc.controllerLock.Unlock()
logDebugf("Poller controller stopped, exiting")
return
}
if pc.httpPoller == nil {
pc.controllerLock.Unlock()
logErrorf("CCCP poller has exited for http fallback but no http poller is configured, retrying CCCP")
continue
}
pc.activeController = pc.httpPoller
pc.controllerLock.Unlock()
pc.httpPoller.DoLoop()
}
}
// Stop should never be called more than once.
func (pc *pollerController) Stop() {
pc.controllerLock.Lock()
pc.stopped = true
controller := pc.activeController
pc.controllerLock.Unlock()
if controller != nil {
controller.Stop()
}
}
func (pc *pollerController) Done() chan struct{} {
pc.controllerLock.Lock()
controller := pc.activeController
pc.controllerLock.Unlock()
if controller == nil {
return nil
}
return controller.Done()
}
type pollerErrorProvider interface {
PollerError() error
}
// PollerError surfaces any error of the underlying poller is currently in an error state.
func (pc *pollerController) PollerError() error {
pc.controllerLock.Lock()
controller := pc.activeController
pc.controllerLock.Unlock()
if controller == nil {
return nil
}
return controller.Error()
}
func (pc *pollerController) ForceHTTPPoller() {
if pc.httpPoller == nil {
logErrorf("Attempting to force http poller but no http poller is configured")
return
}
if !pc.httpPoller.CanPoll() {
logDebugf("Attempting to force http poller but there are no http endpoints to poll")
return
}
go func() {
if atomic.LoadUint32(&pc.bucketConfigSeen) == 1 {
logInfof("Config already seen, not forcing HTTP")
// If we've seen a config already then either cccp or http polling have managed to fetch a config and
// bucket type can't have changed so there's no reason to fallback.
return
}
pc.controllerLock.Lock()
if pc.stopped || pc.activeController == nil {
// If active controller is nil at this point then something strange is happening, we're trying to force
// http polling at the same time as we've received a config via http polling and are attempting to reset to
// use cccp polling (which means that the server must support cccp). If this happens let's just let
// cccp start up.
pc.controllerLock.Unlock()
return
}
if pc.activeController == pc.cccpPoller {
logInfof("Stopping CCCP poller for HTTP polling takeover")
pc.activeController = nil
pc.controllerLock.Unlock()
pc.cccpPoller.Stop()
return
}
pc.controllerLock.Unlock()
}()
}
func isPollingFallbackError(err error, bucket string) bool {
if bucket == "" {
return false
}
return errors.Is(err, ErrDocumentNotFound) || errors.Is(err, ErrUnsupportedOperation) ||
errors.Is(err, errNoCCCPHosts) || errors.Is(err, ErrBucketNotFound)
}
gocbcore-10.2.3/pollercontroller_test.go 0000664 0000000 0000000 00000007472 14417540156 0020370 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"time"
"unsafe"
"github.com/couchbase/gocbcore/v10/memd"
)
// This test tests that after calling stop then force http poller will not attempt to do work.
func (suite *UnitTestSuite) TestPollerControllerForceHTTPAndStopRace() {
ccp := &cccpConfigController{
looperStopSig: make(chan struct{}),
looperDoneSig: make(chan struct{}),
}
cliMux := &httpClientMux{
mgmtEpList: []routeEndpoint{
{
Address: "localhost:8091",
},
},
}
htt := &httpConfigController{
baseHTTPConfigController: &baseHTTPConfigController{
looperStopSig: make(chan struct{}),
looperDoneSig: make(chan struct{}),
},
muxer: &httpMux{
muxPtr: unsafe.Pointer(cliMux),
},
}
poller := newPollerController(ccp, htt, &configManagementComponent{}, func(err error) bool {
return false
})
poller.activeController = ccp
poller.Stop()
poller.ForceHTTPPoller()
suite.Assert().Equal(poller.cccpPoller, poller.activeController)
}
// This test tests the scenario where ForceHTTPPoller and OnNewRouteConfig deadlock.
// This can happen when there are 2+ connections and one successfully bootstraps whilst the
// other fails with bucket not found. If cccp successfully gets a config at the same time
// as the bucket not found connection returns up the stack to ForceHTTPPoller then a deadlock
// can occur where ForceHTTPPoller is waiting for cccp to complete but cccp is blocking by waiting for
// the controllerLock lock in OnNewRouteConfig, which is already held by ForceHTTPPoller.
func (suite *UnitTestSuite) TestPollerControllerForceHTTPAndNewConfig() {
config, err := suite.LoadRawTestDataset("bucket_config_with_external_addresses")
suite.Require().Nil(err)
pipeline := newPipeline(routeEndpoint{Address: "127.0.0.1:11210"}, 1, 10, nil)
muxer := new(mockDispatcher)
muxer.On("PipelineSnapshot").Return(&pipelineSnapshot{
state: &kvMuxState{
routeCfg: routeConfig{
revID: 1,
bktType: bktTypeCouchbase,
},
pipelines: []*memdPipeline{
pipeline,
},
},
idx: 0,
}, nil)
cfgMgr := &configManagementComponent{
currentConfig: &routeConfig{
revID: -1,
},
}
ccp := &cccpConfigController{
looperStopSig: make(chan struct{}),
looperDoneSig: make(chan struct{}),
cfgMgr: cfgMgr,
muxer: muxer,
confCccpPollPeriod: 10 * time.Second,
confCccpMaxWait: 5 * time.Second,
isFallbackErrorFn: func(err error) bool {
return false
},
}
cliMux := &httpClientMux{
mgmtEpList: []routeEndpoint{
{
Address: "localhost:8091",
},
},
}
htt := &httpConfigController{
baseHTTPConfigController: &baseHTTPConfigController{
looperStopSig: make(chan struct{}),
looperDoneSig: make(chan struct{}),
},
muxer: &httpMux{
muxPtr: unsafe.Pointer(cliMux),
},
}
poller := newPollerController(ccp, htt, cfgMgr, func(err error) bool {
return false
})
poller.activeController = ccp
go poller.Run()
c := &memdOpConsumer{
parent: pipeline.queue,
isClosed: false,
}
req := pipeline.queue.pop(c)
suite.Require().Equal(req.Command, memd.CmdGetClusterConfig)
req.tryCallback(&memdQResponse{
Packet: &memd.Packet{
Value: config,
},
}, nil)
poller.ForceHTTPPoller()
// Let ForceHTTPPoller take the lock, have hit the cccp poller done channel, and restarted cccp.
time.Sleep(50 * time.Millisecond)
// This will hang if there's a deadlock between ForceHTTPPoller and OnNewRouteConfig within the poller controller.
suite.Assert().Nil(poller.PollerError())
// The ForceHTTPPoller should pick up that the poller has seen a config whilst it was waiting on the cccp done
// channel and start cccp back up again.
suite.Assert().Equal(poller.cccpPoller, poller.activeController)
poller.Stop()
select {
case <-poller.Done():
case <-time.After(2 * time.Second):
suite.T().Fatalf("Poller controller did not halt in required time")
}
}
gocbcore-10.2.3/querystreamer.go 0000664 0000000 0000000 00000007732 14417540156 0016637 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"errors"
"io"
"sync"
)
// QueryResult allows access to the results of a N1QL query.
type queryStreamer struct {
metaDataBytes []byte
err error
lock sync.Mutex
stream io.ReadCloser
streamer *rowStreamer
}
func newQueryStreamer(stream io.ReadCloser, rowsAttrib string) (*queryStreamer, error) {
rowStreamer, err := newRowStreamer(stream, rowsAttrib)
if err != nil {
closeErr := stream.Close()
if closeErr != nil {
logDebugf("query stream close failed after error: %s", closeErr)
}
return nil, err
}
return &queryStreamer{
stream: stream,
streamer: rowStreamer,
}, nil
}
// NextRow returns the next row from the results, returning nil when the rows are exhausted.
func (r *queryStreamer) NextRow() []byte {
if r.streamer == nil {
return nil
}
rowBytes, err := r.streamer.NextRowBytes()
if err != nil {
r.finishWithError(err)
return nil
}
// Check if there were any rows left
if rowBytes == nil {
r.finishWithoutError()
return nil
}
return rowBytes
}
// Err returns any errors that have occurred on the stream
func (r *queryStreamer) Err() error {
r.lock.Lock()
err := r.err
r.lock.Unlock()
return err
}
// EarlyMetadata returns the value (or nil) of an attribute from a query metadata before the query has completed.
func (r *queryStreamer) EarlyMetadata(key string) json.RawMessage {
return r.streamer.EarlyAttrib(key)
}
func (r *queryStreamer) finishWithoutError() {
// Lets finalize the streamer so we Get the meta-data
metaDataBytes, err := r.streamer.Finalize()
if err != nil {
r.finishWithError(err)
return
}
// Streamer is no longer valid now that it's been Finalized
r.streamer = nil
// Close the stream now that we are done with it
err = r.stream.Close()
if err != nil {
logWarnf("query stream close failed after meta-data: %s", err)
}
// The stream itself is no longer valid
r.lock.Lock()
r.stream = nil
r.lock.Unlock()
r.metaDataBytes = metaDataBytes
}
func (r *queryStreamer) finishWithError(err error) {
// Lets record the error that happened
r.err = err
// Our streamer is invalidated as soon as an error occurs
r.streamer = nil
// Lets close the underlying stream
closeErr := r.stream.Close()
if closeErr != nil {
// We log this at debug level, but its almost always going to be an
// error since thats the most likely reason we are in finishWithError
logDebugf("query stream close failed after error: %s", closeErr)
}
// The stream itself is now no longer valid
r.stream = nil
}
// Close marks the results as closed, returning any errors that occurred during reading the results.
func (r *queryStreamer) Close() error {
// If an error occurred before, we should return that (forever)
err := r.Err()
if err != nil {
return err
}
r.lock.Lock()
stream := r.stream
r.lock.Unlock()
// If the stream is already closed, we can imply that no error occurred
if stream == nil {
return nil
}
return stream.Close()
}
// One assigns the first value from the results into the value pointer.
// It will close the results but not before iterating through all remaining
// results, as such this should only be used for very small resultsets - ideally
// of, at most, length 1.
func (r *queryStreamer) One() ([]byte, error) {
rowBytes := r.NextRow()
if rowBytes == nil {
if r.Err() == nil {
return nil, errors.New("no rows available")
}
return nil, r.Close()
}
// Read any remaining rows
for r.NextRow() != nil {
// skip
}
// If an error occurred during the streaming, we need to
// return that, and make sure the result is closed
err := r.Err()
if err != nil {
return nil, err
}
return rowBytes, nil
}
func (r *queryStreamer) MetaData() ([]byte, error) {
if r.streamer != nil {
return nil, errors.New("the result must be closed before accessing the meta-data")
}
if r.metaDataBytes == nil {
return nil, errors.New("an error occurred during querying which has made the meta-data unavailable")
}
return r.metaDataBytes, nil
}
gocbcore-10.2.3/retry.go 0000664 0000000 0000000 00000032616 14417540156 0015073 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"math"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
// RetryRequest is a request that can possibly be retried.
type RetryRequest interface {
RetryAttempts() uint32
Identifier() string
Idempotent() bool
RetryReasons() []RetryReason
retryStrategy() RetryStrategy
recordRetryAttempt(reason RetryReason)
}
// RetryReason represents the reason for an operation possibly being retried.
type RetryReason interface {
AllowsNonIdempotentRetry() bool
AlwaysRetry() bool
Description() string
}
type retryReason struct {
allowsNonIdempotentRetry bool
alwaysRetry bool
description string
}
func (rr retryReason) AllowsNonIdempotentRetry() bool {
return rr.allowsNonIdempotentRetry
}
func (rr retryReason) AlwaysRetry() bool {
return rr.alwaysRetry
}
func (rr retryReason) Description() string {
return rr.description
}
func (rr retryReason) String() string {
return rr.description
}
func (rr retryReason) MarshalJSON() ([]byte, error) {
return json.Marshal(rr.description)
}
var (
// UnknownRetryReason indicates that the operation failed for an unknown reason.
UnknownRetryReason = retryReason{allowsNonIdempotentRetry: false, alwaysRetry: false, description: "UNKNOWN"}
// SocketNotAvailableRetryReason indicates that the operation failed because the underlying socket was not available.
SocketNotAvailableRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "SOCKET_NOT_AVAILABLE"}
// ServiceNotAvailableRetryReason indicates that the operation failed because the requested service was not available.
ServiceNotAvailableRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "SERVICE_NOT_AVAILABLE"}
// NodeNotAvailableRetryReason indicates that the operation failed because the requested node was not available.
NodeNotAvailableRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "NODE_NOT_AVAILABLE"}
// KVNotMyVBucketRetryReason indicates that the operation failed because it was sent to the wrong node for the vbucket.
KVNotMyVBucketRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: true, description: "KV_NOT_MY_VBUCKET"}
// KVCollectionOutdatedRetryReason indicates that the operation failed because the collection ID on the request is outdated.
KVCollectionOutdatedRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: true, description: "KV_COLLECTION_OUTDATED"}
// KVErrMapRetryReason indicates that the operation failed for an unsupported reason but the KV error map indicated
// that the operation can be retried.
KVErrMapRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "KV_ERROR_MAP_RETRY_INDICATED"}
// KVLockedRetryReason indicates that the operation failed because the document was locked.
KVLockedRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "KV_LOCKED"}
// KVTemporaryFailureRetryReason indicates that the operation failed because of a temporary failure.
KVTemporaryFailureRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "KV_TEMPORARY_FAILURE"}
// KVSyncWriteInProgressRetryReason indicates that the operation failed because a sync write is in progress.
KVSyncWriteInProgressRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "KV_SYNC_WRITE_IN_PROGRESS"}
// KVSyncWriteRecommitInProgressRetryReason indicates that the operation failed because a sync write recommit is in progress.
KVSyncWriteRecommitInProgressRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "KV_SYNC_WRITE_RE_COMMIT_IN_PROGRESS"}
// ServiceResponseCodeIndicatedRetryReason indicates that the operation failed and the service responded stating that
// the request should be retried.
ServiceResponseCodeIndicatedRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "SERVICE_RESPONSE_CODE_INDICATED"}
// SocketCloseInFlightRetryReason indicates that the operation failed because the socket was closed whilst the operation
// was in flight.
SocketCloseInFlightRetryReason = retryReason{allowsNonIdempotentRetry: false, alwaysRetry: false, description: "SOCKET_CLOSED_WHILE_IN_FLIGHT"}
// PipelineOverloadedRetryReason indicates that the operation failed because the pipeline queue was full.
PipelineOverloadedRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: true, description: "PIPELINE_OVERLOADED"}
// CircuitBreakerOpenRetryReason indicates that the operation failed because the circuit breaker for the underlying socket was open.
CircuitBreakerOpenRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "CIRCUIT_BREAKER_OPEN"}
// QueryIndexNotFoundRetryReason indicates that the operation failed to to a missing query index
QueryIndexNotFoundRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "QUERY_INDEX_NOT_FOUND"}
// QueryPreparedStatementFailureRetryReason indicates that the operation failed due to a prepared statement failure
QueryPreparedStatementFailureRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "QUERY_PREPARED_STATEMENT_FAILURE"}
// QueryErrorRetryable indicates that the operation is retryable as indicated by the query engine.
QueryErrorRetryable = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "QUERY_ERROR_RETRYABLE"}
// AnalyticsTemporaryFailureRetryReason indicates that an analytics operation failed due to a temporary failure
AnalyticsTemporaryFailureRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "ANALYTICS_TEMPORARY_FAILURE"}
// SearchTooManyRequestsRetryReason indicates that a search operation failed due to too many requests
SearchTooManyRequestsRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "SEARCH_TOO_MANY_REQUESTS"}
// NotReadyRetryReason indicates that the WaitUntilReady operation is not ready
NotReadyRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: true, description: "NOT_READY"}
// NoPipelineSnapshotRetryReason indicates that there was no pipeline snapshot available
NoPipelineSnapshotRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "NO_PIPELINE_SNAPSHOT"}
// BucketNotReadyReason indicates that the user has priviledges to access the bucket but the bucket doesn't exist
// or is in warm up.
// Uncommitted: This API may change in the future.
BucketNotReadyReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "BUCKET_NOT_FOUND"}
// ConnectionErrorRetryReason indicates that there were errors reported by underlying connections.
// Check server ports and cluster encryption setting.
ConnectionErrorRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false, description: "CONNECTION_ERROR"}
// MemdWriteFailure indicates that the operation failed because the write failed on the connection.
MemdWriteFailure = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: true, description: "MEMD_WRITE_FAILURE"}
// CredentialsFetchFailedRetryReason indicates that the operation failed because the AuthProvider return an error for credentials.
// Uncommitted: This API may change in the future.
CredentialsFetchFailedRetryReason = retryReason{allowsNonIdempotentRetry: true, alwaysRetry: true, description: "CREDENTIALS_FETCH_FAILED"}
)
// MaybeRetryRequest will possibly retry a request according to the strategy belonging to the request.
// It will use the reason to determine whether or not the failure reason is one that can be retried.
func (agent *Agent) MaybeRetryRequest(req RetryRequest, reason RetryReason) (bool, time.Time) {
return retryOrchMaybeRetry(req, reason)
}
// RetryAction is used by a RetryStrategy to calculate the duration to wait before retrying an operation.
// Returning a value of 0 indicates to not retry.
type RetryAction interface {
Duration() time.Duration
}
// NoRetryRetryAction represents an action that indicates to not retry.
type NoRetryRetryAction struct {
}
// Duration is the length of time to wait before retrying an operation.
func (ra *NoRetryRetryAction) Duration() time.Duration {
return 0
}
// WithDurationRetryAction represents an action that indicates to retry with a given duration.
type WithDurationRetryAction struct {
WithDuration time.Duration
}
// Duration is the length of time to wait before retrying an operation.
func (ra *WithDurationRetryAction) Duration() time.Duration {
return ra.WithDuration
}
// RetryStrategy is to determine if an operation should be retried, and if so how long to wait before retrying.
type RetryStrategy interface {
RetryAfter(req RetryRequest, reason RetryReason) RetryAction
}
// retryOrchMaybeRetry will possibly retry an operation according to the strategy belonging to the request.
// It will use the reason to determine whether or not the failure reason is one that can be retried.
func retryOrchMaybeRetry(req RetryRequest, reason RetryReason) (bool, time.Time) {
if reason.AlwaysRetry() {
duration := ControlledBackoff(req.RetryAttempts())
logDebugf("Will retry request. Backoff=%s, OperationID=%s. Reason=%s", duration, req.Identifier(), reason)
req.recordRetryAttempt(reason)
return true, time.Now().Add(duration)
}
retryStrategy := req.retryStrategy()
if retryStrategy == nil {
return false, time.Time{}
}
action := retryStrategy.RetryAfter(req, reason)
if action == nil {
logDebugf("Won't retry request. OperationID=%s. Reason=%s", req.Identifier(), reason)
return false, time.Time{}
}
duration := action.Duration()
if duration == 0 {
logDebugf("Won't retry request. OperationID=%s. Reason=%s", req.Identifier(), reason)
return false, time.Time{}
}
logDebugf("Will retry request. Backoff=%s, OperationID=%s. Reason=%s", duration, req.Identifier(), reason)
req.recordRetryAttempt(reason)
return true, time.Now().Add(duration)
}
// failFastRetryStrategy represents a strategy that will never retry.
type failFastRetryStrategy struct {
}
// newFailFastRetryStrategy returns a new FailFastRetryStrategy.
func newFailFastRetryStrategy() *failFastRetryStrategy {
return &failFastRetryStrategy{}
}
// RetryAfter calculates and returns a RetryAction describing how long to wait before retrying an operation.
func (rs *failFastRetryStrategy) RetryAfter(req RetryRequest, reason RetryReason) RetryAction {
return &NoRetryRetryAction{}
}
// BackoffCalculator is used by retry strategies to calculate backoff durations.
type BackoffCalculator func(retryAttempts uint32) time.Duration
// BestEffortRetryStrategy represents a strategy that will keep retrying until it succeeds (or the caller times out
// the request).
type BestEffortRetryStrategy struct {
backoffCalculator BackoffCalculator
}
// NewBestEffortRetryStrategy returns a new BestEffortRetryStrategy which will use the supplied calculator function
// to calculate retry durations. If calculator is nil then ControlledBackoff will be used.
func NewBestEffortRetryStrategy(calculator BackoffCalculator) *BestEffortRetryStrategy {
if calculator == nil {
calculator = ControlledBackoff
}
return &BestEffortRetryStrategy{backoffCalculator: calculator}
}
// RetryAfter calculates and returns a RetryAction describing how long to wait before retrying an operation.
func (rs *BestEffortRetryStrategy) RetryAfter(req RetryRequest, reason RetryReason) RetryAction {
if req.Idempotent() || reason.AllowsNonIdempotentRetry() {
return &WithDurationRetryAction{WithDuration: rs.backoffCalculator(req.RetryAttempts())}
}
return &NoRetryRetryAction{}
}
// ExponentialBackoff calculates a backoff time duration from the retry attempts on a given request.
func ExponentialBackoff(min, max time.Duration, backoffFactor float64) BackoffCalculator {
var minBackoff float64 = 1000000 // 1 Millisecond
var maxBackoff float64 = 500000000 // 500 Milliseconds
var factor float64 = 2
if min > 0 {
minBackoff = float64(min)
}
if max > 0 {
maxBackoff = float64(max)
}
if backoffFactor > 0 {
factor = backoffFactor
}
return func(retryAttempts uint32) time.Duration {
backoff := minBackoff * (math.Pow(factor, float64(retryAttempts)))
if backoff > maxBackoff {
backoff = maxBackoff
}
if backoff < minBackoff {
backoff = minBackoff
}
return time.Duration(backoff)
}
}
// ControlledBackoff calculates a backoff time duration from the retry attempts on a given request.
func ControlledBackoff(retryAttempts uint32) time.Duration {
switch retryAttempts {
case 0:
return 1 * time.Millisecond
case 1:
return 10 * time.Millisecond
case 2:
return 50 * time.Millisecond
case 3:
return 100 * time.Millisecond
case 4:
return 500 * time.Millisecond
default:
return 1000 * time.Millisecond
}
}
var idempotentOps = map[memd.CmdCode]bool{
memd.CmdGet: true,
memd.CmdGetReplica: true,
memd.CmdGetMeta: true,
memd.CmdSubDocGet: true,
memd.CmdSubDocExists: true,
memd.CmdSubDocGetCount: true,
memd.CmdNoop: true,
memd.CmdStat: true,
memd.CmdGetRandom: true,
memd.CmdCollectionsGetID: true,
memd.CmdCollectionsGetManifest: true,
memd.CmdGetClusterConfig: true,
memd.CmdObserve: true,
memd.CmdObserveSeqNo: true,
}
gocbcore-10.2.3/retry_test.go 0000664 0000000 0000000 00000026460 14417540156 0016132 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"fmt"
"reflect"
"testing"
"time"
)
type mockRetryRequest struct {
attempts uint32
identifier string
idempotent bool
reasons []RetryReason
cancelFunc func() bool
strategy RetryStrategy
}
func (mgr *mockRetryRequest) retryStrategy() RetryStrategy {
return mgr.strategy
}
func (mgr *mockRetryRequest) RetryAttempts() uint32 {
return mgr.attempts
}
func (mgr *mockRetryRequest) Identifier() string {
return mgr.identifier
}
func (mgr *mockRetryRequest) Idempotent() bool {
return mgr.idempotent
}
func (mgr *mockRetryRequest) RetryReasons() []RetryReason {
return mgr.reasons
}
func (mgr *mockRetryRequest) recordRetryAttempt(reason RetryReason) {
mgr.attempts++
for _, foundReason := range mgr.reasons {
if foundReason == reason {
return
}
}
mgr.reasons = append(mgr.reasons, reason)
}
func (mgr *mockRetryRequest) setCancelRetry(cancelFunc func() bool) {
mgr.cancelFunc = cancelFunc
}
type mockRetryStrategy struct {
retried bool
action RetryAction
}
func (mrs *mockRetryStrategy) RetryAfter(req RetryRequest, reason RetryReason) RetryAction {
mrs.retried = true
return mrs.action
}
func mockBackoffCalculator(retryAttempts uint32) time.Duration {
return time.Millisecond * time.Duration(retryAttempts)
}
func (suite *StandardTestSuite) TestRetryOrchestrator() {
type test struct {
name string
shouldRetry bool
retryReason RetryReason
request *mockRetryRequest
expectedAttempts uint32
retryReasonsLen int
}
tests := map[RetryStrategy][]test{
NewBestEffortRetryStrategy(nil): {
{
name: "not idempotent request, allowsNonIdempotentRetry: false, alwaysRetry: false",
shouldRetry: false,
retryReason: &retryReason{allowsNonIdempotentRetry: false, alwaysRetry: false},
request: &mockRetryRequest{attempts: 0},
expectedAttempts: 0,
retryReasonsLen: 0,
},
{
name: "idempotent request, allowsNonIdempotentRetry: false, alwaysRetry: false",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: false, alwaysRetry: false},
request: &mockRetryRequest{attempts: 0, idempotent: true},
expectedAttempts: 3,
retryReasonsLen: 1,
},
{
name: "not idempotent request, allowsNonIdempotentRetry: true, alwaysRetry: false",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false},
request: &mockRetryRequest{attempts: 0},
expectedAttempts: 3,
retryReasonsLen: 1,
},
{
name: "idempotent request, allowsNonIdempotentRetry: true, alwaysRetry: false",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false},
request: &mockRetryRequest{attempts: 0, idempotent: true},
expectedAttempts: 3,
retryReasonsLen: 1,
},
{
name: "not idempotent request, allowsNonIdempotentRetry: true, alwaysRetry: true",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: true, alwaysRetry: true},
request: &mockRetryRequest{attempts: 0},
expectedAttempts: 3,
retryReasonsLen: 1,
},
{
name: "idempotent request, allowsNonIdempotentRetry: true, alwaysRetry: true",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: true, alwaysRetry: true},
request: &mockRetryRequest{attempts: 0, idempotent: true},
expectedAttempts: 3,
retryReasonsLen: 1,
},
{
name: "not idempotent request, allowsNonIdempotentRetry: false, alwaysRetry: true",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: false, alwaysRetry: true},
request: &mockRetryRequest{attempts: 0},
expectedAttempts: 3,
retryReasonsLen: 1,
},
{
name: "idempotent request, allowsNonIdempotentRetry: false, alwaysRetry: true",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: false, alwaysRetry: true},
request: &mockRetryRequest{attempts: 0, idempotent: true},
expectedAttempts: 3,
retryReasonsLen: 1,
},
},
newFailFastRetryStrategy(): {
{
name: "not idempotent request, allowsNonIdempotentRetry: false, alwaysRetry: false",
shouldRetry: false,
retryReason: &retryReason{allowsNonIdempotentRetry: false, alwaysRetry: false},
request: &mockRetryRequest{attempts: 0},
expectedAttempts: 0,
retryReasonsLen: 0,
},
{
name: "idempotent request, allowsNonIdempotentRetry: false, alwaysRetry: false",
shouldRetry: false,
retryReason: &retryReason{allowsNonIdempotentRetry: false, alwaysRetry: false},
request: &mockRetryRequest{attempts: 0, idempotent: true},
expectedAttempts: 0,
retryReasonsLen: 0,
},
{
name: "not idempotent request, allowsNonIdempotentRetry: true, alwaysRetry: false",
shouldRetry: false,
retryReason: &retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false},
request: &mockRetryRequest{attempts: 0},
expectedAttempts: 0,
retryReasonsLen: 0,
},
{
name: "idempotent request, allowsNonIdempotentRetry: true, alwaysRetry: false",
shouldRetry: false,
retryReason: &retryReason{allowsNonIdempotentRetry: true, alwaysRetry: false},
request: &mockRetryRequest{attempts: 0, idempotent: true},
expectedAttempts: 0,
retryReasonsLen: 0,
},
{
name: "not idempotent request, allowsNonIdempotentRetry: true, alwaysRetry: true",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: true, alwaysRetry: true},
request: &mockRetryRequest{attempts: 0},
expectedAttempts: 3,
retryReasonsLen: 1,
},
{
name: "idempotent request, allowsNonIdempotentRetry: true, alwaysRetry: true",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: true, alwaysRetry: true},
request: &mockRetryRequest{attempts: 0, idempotent: true},
expectedAttempts: 3,
retryReasonsLen: 1,
},
{
name: "not idempotent request, allowsNonIdempotentRetry: false, alwaysRetry: true",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: false, alwaysRetry: true},
request: &mockRetryRequest{attempts: 0},
expectedAttempts: 3,
retryReasonsLen: 1,
},
{
name: "idempotent request, allowsNonIdempotentRetry: false, alwaysRetry: true",
shouldRetry: true,
retryReason: &retryReason{allowsNonIdempotentRetry: false, alwaysRetry: true},
request: &mockRetryRequest{attempts: 0, idempotent: true},
expectedAttempts: 3,
retryReasonsLen: 1,
},
},
}
for strategy, rsTests := range tests {
stratTyp := reflect.ValueOf(strategy).Type()
for _, tt := range rsTests {
suite.T().Run(fmt.Sprintf("%s - %s", stratTyp, tt.name), func(t *testing.T) {
// Copy it and add the strategy
baseReq := *tt.request
req := &baseReq
req.strategy = strategy
totalWaitTime := time.Duration(0)
for {
shouldRetry, retryTime := retryOrchMaybeRetry(req, tt.retryReason)
if shouldRetry != tt.shouldRetry {
suite.T().Fatalf("Expected retried to be %v, got %v", tt.shouldRetry, shouldRetry)
}
// No need to retry, just break
if !shouldRetry {
break
}
waitDuration := retryTime.Sub(time.Now())
totalWaitTime += waitDuration
if totalWaitTime >= 50*time.Millisecond {
break
}
}
if tt.expectedAttempts != req.RetryAttempts() {
suite.T().Fatalf("Expected retries to be %d, was %d", tt.expectedAttempts, req.RetryAttempts())
}
if tt.retryReasonsLen != len(req.RetryReasons()) {
suite.T().Fatalf("Expected reasons to be %d, was %d", tt.retryReasonsLen, len(req.RetryReasons()))
}
})
}
}
}
type cancellationRetryStrategy struct {
}
func (crs *cancellationRetryStrategy) RetryAfter(req RetryRequest, reason RetryReason) RetryAction {
return &WithDurationRetryAction{WithDuration: 50 * time.Millisecond}
}
func (suite *StandardTestSuite) TestControlledBackoff() {
type test struct {
attempts uint32
expectedBackoff time.Duration
}
tests := []test{
{
attempts: 0,
expectedBackoff: 1 * time.Millisecond,
},
{
attempts: 1,
expectedBackoff: 10 * time.Millisecond,
},
{
attempts: 2,
expectedBackoff: 50 * time.Millisecond,
},
{
attempts: 3,
expectedBackoff: 100 * time.Millisecond,
},
{
attempts: 4,
expectedBackoff: 500 * time.Millisecond,
},
{
attempts: 5,
expectedBackoff: 1000 * time.Millisecond,
},
{
attempts: 6,
expectedBackoff: 1000 * time.Millisecond,
},
}
for _, tt := range tests {
backoff := ControlledBackoff(tt.attempts)
if backoff != tt.expectedBackoff {
suite.T().Fatalf("Expected backoff to be %s but was %s", tt.expectedBackoff.String(), backoff.String())
}
}
}
func (suite *StandardTestSuite) TestExponentialBackoff() {
type test struct {
attempts uint32
expectedBackoff time.Duration
}
tests := []test{
{
attempts: 0,
expectedBackoff: 1 * time.Millisecond,
},
{
attempts: 1,
expectedBackoff: 2 * time.Millisecond,
},
{
attempts: 2,
expectedBackoff: 4 * time.Millisecond,
},
{
attempts: 3,
expectedBackoff: 8 * time.Millisecond,
},
{
attempts: 4,
expectedBackoff: 16 * time.Millisecond,
},
{
attempts: 5,
expectedBackoff: 32 * time.Millisecond,
},
{
attempts: 6,
expectedBackoff: 64 * time.Millisecond,
},
{
attempts: 7,
expectedBackoff: 128 * time.Millisecond,
},
{
attempts: 8,
expectedBackoff: 256 * time.Millisecond,
},
{
attempts: 9,
expectedBackoff: 500 * time.Millisecond,
},
{
attempts: 10,
expectedBackoff: 500 * time.Millisecond,
},
}
for _, tt := range tests {
calc := ExponentialBackoff(0, 0, 0)
backoff := calc(tt.attempts)
if backoff != tt.expectedBackoff {
suite.T().Fatalf("Expected backoff to be %s but was %s", tt.expectedBackoff.String(), backoff.String())
}
}
}
func (suite *StandardTestSuite) TestExponentialBackoffNonDefaults() {
type test struct {
attempts uint32
expectedBackoff time.Duration
}
tests := []test{
{
attempts: 0,
expectedBackoff: 10 * time.Millisecond,
},
{
attempts: 1,
expectedBackoff: 30 * time.Millisecond,
},
{
attempts: 2,
expectedBackoff: 90 * time.Millisecond,
},
{
attempts: 3,
expectedBackoff: 270 * time.Millisecond,
},
{
attempts: 4,
expectedBackoff: 810 * time.Millisecond,
},
{
attempts: 5,
expectedBackoff: 1000 * time.Millisecond,
},
{
attempts: 6,
expectedBackoff: 1000 * time.Millisecond,
},
}
for _, tt := range tests {
calc := ExponentialBackoff(10*time.Millisecond, 1000*time.Millisecond, 3)
backoff := calc(tt.attempts)
if backoff != tt.expectedBackoff {
suite.T().Fatalf("Expected backoff to be %s but was %s", tt.expectedBackoff.String(), backoff.String())
}
}
}
gocbcore-10.2.3/routeconfig.go 0000664 0000000 0000000 00000007775 14417540156 0016262 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"bytes"
"fmt"
)
type routeEndpoints struct {
SSLEndpoints []routeEndpoint
NonSSLEndpoints []routeEndpoint
}
type routeEndpoint struct {
Address string
IsSeedNode bool
}
type routeConfig struct {
revID int64
revEpoch int64
uuid string
name string
bktType bucketType
kvServerList routeEndpoints
capiEpList routeEndpoints
mgmtEpList routeEndpoints
n1qlEpList routeEndpoints
ftsEpList routeEndpoints
cbasEpList routeEndpoints
eventingEpList routeEndpoints
gsiEpList routeEndpoints
backupEpList routeEndpoints
vbMap *vbucketMap
ketamaMap *ketamaContinuum
clusterCapabilitiesVer []int
clusterCapabilities map[string][]string
bucketCapabilities []string
bucketCapabilitiesVer string
}
func (config *routeConfig) DebugString() string {
var buffer bytes.Buffer
buffer.WriteString(fmt.Sprintf("Revision ID: %d\n", config.revID))
buffer.WriteString(fmt.Sprintf("Revision Epoch: %d\n", config.revEpoch))
if config.name != "" {
fmt.Fprintf(&buffer, "Bucket: %s\n", config.name)
}
addEps := func(title string, eps routeEndpoints) {
fmt.Fprintf(&buffer, "%s Eps:\n", title)
fmt.Fprintln(&buffer, " TLS:")
for _, ep := range eps.NonSSLEndpoints {
fmt.Fprintf(&buffer, " - %s seed: %t\n", ep.Address, ep.IsSeedNode)
}
fmt.Fprintln(&buffer, " Non-TLS:")
for _, ep := range eps.SSLEndpoints {
fmt.Fprintf(&buffer, " - %s seed: %t\n", ep.Address, ep.IsSeedNode)
}
}
addEps("Capi", config.capiEpList)
addEps("Mgmt", config.mgmtEpList)
addEps("N1ql", config.n1qlEpList)
addEps("FTS", config.ftsEpList)
addEps("CBAS", config.cbasEpList)
addEps("Eventing", config.eventingEpList)
addEps("GSI", config.gsiEpList)
addEps("Backup", config.backupEpList)
if config.vbMap != nil {
fmt.Fprintln(&buffer, "VBMap:")
fmt.Fprintf(&buffer, "%+v\n", config.vbMap)
} else {
fmt.Fprintln(&buffer, "VBMap: not-used")
}
if config.ketamaMap != nil {
fmt.Fprintln(&buffer, "KetamaMap:")
fmt.Fprintf(&buffer, "%+v\n", config.ketamaMap)
} else {
fmt.Fprintln(&buffer, "KetamaMap: not-used")
}
return buffer.String()
}
func (config *routeConfig) IsValid() bool {
if (len(config.kvServerList.SSLEndpoints) == 0 || len(config.mgmtEpList.SSLEndpoints) == 0) &&
(len(config.kvServerList.NonSSLEndpoints) == 0 || len(config.mgmtEpList.NonSSLEndpoints) == 0) {
return false
}
switch config.bktType {
case bktTypeCouchbase:
return config.vbMap != nil && config.vbMap.IsValid()
case bktTypeMemcached:
return config.ketamaMap != nil && config.ketamaMap.IsValid()
case bktTypeNone:
return true
default:
return false
}
}
func (config *routeConfig) IsGCCCPConfig() bool {
return config.bktType == bktTypeNone
}
func (config *routeConfig) ContainsClusterCapability(version int, category, capability string) bool {
caps := config.clusterCapabilities
capsVer := config.clusterCapabilitiesVer
if len(capsVer) == 0 || caps == nil {
return false
}
if capsVer[0] == version {
for cat, catCapabilities := range caps {
switch cat {
case category:
for _, capa := range catCapabilities {
switch capa {
case capability:
return true
}
}
}
}
}
return false
}
func (config *routeConfig) ContainsBucketCapability(needleCap string) bool {
for _, capa := range config.bucketCapabilities {
if capa == needleCap {
return true
}
}
return false
}
func (config *routeConfig) IsNewerThan(oldCfg *routeConfig) bool {
if config.revEpoch < oldCfg.revEpoch {
logDebugf("Ignoring new configuration as it has an older revision epoch")
return false
} else if config.revEpoch == oldCfg.revEpoch {
if config.revID == 0 {
logDebugf("Unversioned configuration data, switching.")
} else if config.revID == oldCfg.revID {
logDebugf("Ignoring configuration with identical revision number")
return false
} else if config.revID < oldCfg.revID {
logDebugf("Ignoring new configuration as it has an older revision id")
return false
}
}
return true
}
gocbcore-10.2.3/rowstreamer.go 0000664 0000000 0000000 00000010376 14417540156 0016277 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"errors"
"io"
)
type rowStreamState int
const (
rowStreamStateStart rowStreamState = 0
rowStreamStateRows rowStreamState = 1
rowStreamStatePostRows rowStreamState = 2
rowStreamStateEnd rowStreamState = 3
)
type rowStreamer struct {
decoder *json.Decoder
rowsAttrib string
attribs map[string]json.RawMessage
state rowStreamState
}
func newRowStreamer(stream io.Reader, rowsAttrib string) (*rowStreamer, error) {
decoder := json.NewDecoder(stream)
streamer := &rowStreamer{
decoder: decoder,
rowsAttrib: rowsAttrib,
attribs: make(map[string]json.RawMessage),
state: rowStreamStateStart,
}
if err := streamer.begin(); err != nil {
return nil, err
}
return streamer, nil
}
func (s *rowStreamer) begin() error {
if s.state != rowStreamStateStart {
return errors.New("unexpected parsing state during begin")
}
// Read the opening { for the result
t, err := s.decoder.Token()
if err != nil {
return err
}
if delim, ok := t.(json.Delim); !ok || delim != '{' {
return errors.New("expected an opening brace for the result")
}
for {
if !s.decoder.More() {
// We reached the end of the object
s.state = rowStreamStateEnd
break
}
// Read the attribute name
t, err = s.decoder.Token()
if err != nil {
return err
}
key, keyOk := t.(string)
if !keyOk {
return errors.New("expected an object property name")
}
if key == s.rowsAttrib {
// Read the opening [ for the rows
t, err = s.decoder.Token()
if err != nil {
return err
}
if t == nil {
continue
}
if delim, ok := t.(json.Delim); !ok || delim != '[' {
return errors.New("expected an opening bracket for the rows")
}
s.state = rowStreamStateRows
break
}
// Read the attribute value
var value json.RawMessage
err = s.decoder.Decode(&value)
if err != nil {
return err
}
// Save the attribute for the meta-data
s.attribs[key] = value
}
return nil
}
func (s *rowStreamer) readRow() (json.RawMessage, error) {
if s.state < rowStreamStateRows {
return nil, errors.New("unexpected parsing state during readRow")
}
// If we've already read all rows or rows is null, we return nil
if s.state > rowStreamStateRows {
return nil, nil
}
// If there are no more rows, mark the rows finished and
// return nil to signal that we are at the end
if !s.decoder.More() {
s.state = rowStreamStatePostRows
return nil, nil
}
// Decode this row and return a raw message
var msg json.RawMessage
err := s.decoder.Decode(&msg)
if err != nil {
return nil, err
}
return msg, nil
}
func (s *rowStreamer) end() error {
if s.state < rowStreamStatePostRows {
return errors.New("unexpected parsing state during end")
}
// Check if we've already read everything
if s.state > rowStreamStatePostRows {
return nil
}
// Read the ending ] for the rows
t, err := s.decoder.Token()
if err != nil {
return err
}
if delim, ok := t.(json.Delim); !ok || delim != ']' {
return errors.New("expected an ending bracket for the rows")
}
for {
if !s.decoder.More() {
// We reached the end of the object
s.state = rowStreamStateEnd
break
}
// Read the attribute name
t, err := s.decoder.Token()
if err != nil {
return err
}
key, keyOk := t.(string)
if !keyOk {
return errors.New("expected an object property name")
}
// Read the attribute value
var value json.RawMessage
err = s.decoder.Decode(&value)
if err != nil {
return err
}
// Save the attribute for the meta-data
s.attribs[key] = value
}
return nil
}
func (s *rowStreamer) NextRowBytes() (json.RawMessage, error) {
return s.readRow()
}
func (s *rowStreamer) Finalize() (json.RawMessage, error) {
// Make sure we've read until the end of the object
for {
row, err := s.readRow()
if err != nil {
return nil, err
}
if row == nil {
break
}
}
// Read the rest of the result object
err := s.end()
if err != nil {
return nil, err
}
// Reconstruct the non-rows JSON to a raw message
metaBytes, err := json.Marshal(s.attribs)
if err != nil {
return nil, err
}
return json.RawMessage(metaBytes), nil
}
func (s *rowStreamer) EarlyAttrib(key string) json.RawMessage {
val, ok := s.attribs[key]
if !ok {
return nil
}
return val
}
gocbcore-10.2.3/scram/ 0000775 0000000 0000000 00000000000 14417540156 0014474 5 ustar 00root root 0000000 0000000 gocbcore-10.2.3/scram/scramclient.go 0000664 0000000 0000000 00000017114 14417540156 0017333 0 ustar 00root root 0000000 0000000 // Copyright (c) 2014 - Gustavo Niemeyer
// Copyright (c) 2017 - Couchbase Inc.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package gocbcore
import (
"bytes"
"crypto/hmac"
"crypto/rand"
"encoding/base64"
"fmt"
"hash"
"strconv"
"strings"
)
// Client implements a SCRAM-{SHA-1,etc} client per RFC5802.
// http://tools.ietf.org/html/rfc5802
type Client struct {
newHash func() hash.Hash
user string
pass string
step int
out bytes.Buffer
err error
clientNonce []byte
serverNonce []byte
saltedPass []byte
authMsg bytes.Buffer
}
// NewClient returns a new instance of the SCRAM client.
func NewClient(newHash func() hash.Hash, user, pass string) *Client {
c := &Client{
newHash: newHash,
user: user,
pass: pass,
}
c.out.Grow(256)
c.authMsg.Grow(256)
return c
}
// Out returns the data to be sent to the server in the current step.
func (c *Client) Out() []byte {
if c.out.Len() == 0 {
return nil
}
return c.out.Bytes()
}
// Err returns the error that occurred, or nil if there were no errors.
func (c *Client) Err() error {
return c.err
}
// SetNonce sets the client nonce to the provided value.
// If not set, the nonce is generated automatically out of crypto/rand on the first step.
func (c *Client) SetNonce(nonce []byte) {
c.clientNonce = nonce
}
var escaper = strings.NewReplacer("=", "=3D", ",", "=2C")
// Step processes the incoming data from the server and makes the
// next round of data for the server available via Client.Out.
// Step returns false if there are no errors and more data is
// still expected.
func (c *Client) Step(in []byte) bool {
c.out.Reset()
if c.step > 2 || c.err != nil {
return false
}
c.step++
switch c.step {
case 1:
c.err = c.step1(in)
case 2:
c.err = c.step2(in)
case 3:
c.err = c.step3(in)
}
return !(c.step > 2 || c.err != nil)
}
func (c *Client) step1(in []byte) error {
if len(c.clientNonce) == 0 {
const nonceLen = 6
buf := make([]byte, nonceLen+b64.EncodedLen(nonceLen))
if _, err := rand.Read(buf[:nonceLen]); err != nil {
return fmt.Errorf("cannot read random SCRAM-SHA-1 nonce from operating system: %v", err)
}
c.clientNonce = buf[nonceLen:]
b64.Encode(c.clientNonce, buf[:nonceLen])
}
c.authMsg.WriteString("n=")
if _, err := escaper.WriteString(&c.authMsg, c.user); err != nil {
return err
}
c.authMsg.WriteString(",r=")
c.authMsg.Write(c.clientNonce)
c.out.WriteString("n,,")
c.out.Write(c.authMsg.Bytes())
return nil
}
var b64 = base64.StdEncoding
func (c *Client) step2(in []byte) error {
c.authMsg.WriteByte(',')
c.authMsg.Write(in)
fields := bytes.Split(in, []byte(","))
if len(fields) != 3 {
return fmt.Errorf("expected 3 fields in first SCRAM-SHA-1 server message, got %d: %q", len(fields), in)
}
if !bytes.HasPrefix(fields[0], []byte("r=")) || len(fields[0]) < 2 {
return fmt.Errorf("server sent an invalid SCRAM-SHA-1 nonce: %q", fields[0])
}
if !bytes.HasPrefix(fields[1], []byte("s=")) || len(fields[1]) < 6 {
return fmt.Errorf("server sent an invalid SCRAM-SHA-1 salt: %q", fields[1])
}
if !bytes.HasPrefix(fields[2], []byte("i=")) || len(fields[2]) < 6 {
return fmt.Errorf("server sent an invalid SCRAM-SHA-1 iteration count: %q", fields[2])
}
c.serverNonce = fields[0][2:]
if !bytes.HasPrefix(c.serverNonce, c.clientNonce) {
return fmt.Errorf("server SCRAM-SHA-1 nonce is not prefixed by client nonce: got %q, want %q+\"...\"", c.serverNonce, c.clientNonce)
}
salt := make([]byte, b64.DecodedLen(len(fields[1][2:])))
n, err := b64.Decode(salt, fields[1][2:])
if err != nil {
return fmt.Errorf("cannot decode SCRAM-SHA-1 salt sent by server: %q", fields[1])
}
salt = salt[:n]
iterCount, err := strconv.Atoi(string(fields[2][2:]))
if err != nil {
return fmt.Errorf("server sent an invalid SCRAM-SHA-1 iteration count: %q", fields[2])
}
if err := c.saltPassword(salt, iterCount); err != nil {
return err
}
c.authMsg.WriteString(",c=biws,r=")
c.authMsg.Write(c.serverNonce)
c.out.WriteString("c=biws,r=")
c.out.Write(c.serverNonce)
c.out.WriteString(",p=")
proof, err := c.clientProof()
if err != nil {
return err
}
c.out.Write(proof)
return nil
}
func (c *Client) step3(in []byte) error {
var isv, ise bool
var fields = bytes.Split(in, []byte(","))
if len(fields) == 1 {
isv = bytes.HasPrefix(fields[0], []byte("v="))
ise = bytes.HasPrefix(fields[0], []byte("e="))
}
if ise {
return fmt.Errorf("SCRAM-SHA-1 authentication error: %s", fields[0][2:])
} else if !isv {
return fmt.Errorf("unsupported SCRAM-SHA-1 final message from server: %q", in)
}
sig, err := c.serverSignature()
if err != nil {
return err
}
if !bytes.Equal(sig, fields[0][2:]) {
return fmt.Errorf("cannot authenticate SCRAM-SHA-1 server signature: %q", fields[0][2:])
}
return nil
}
func (c *Client) saltPassword(salt []byte, iterCount int) error {
mac := hmac.New(c.newHash, []byte(c.pass))
if _, err := mac.Write(salt); err != nil {
return err
}
if _, err := mac.Write([]byte{0, 0, 0, 1}); err != nil {
return err
}
ui := mac.Sum(nil)
hi := make([]byte, len(ui))
copy(hi, ui)
for i := 1; i < iterCount; i++ {
mac.Reset()
if _, err := mac.Write(ui); err != nil {
return err
}
mac.Sum(ui[:0])
for j, b := range ui {
hi[j] ^= b
}
}
c.saltedPass = hi
return nil
}
func (c *Client) clientProof() ([]byte, error) {
mac := hmac.New(c.newHash, c.saltedPass)
if _, err := mac.Write([]byte("Client Key")); err != nil {
return nil, err
}
clientKey := mac.Sum(nil)
hash := c.newHash()
if _, err := hash.Write(clientKey); err != nil {
return nil, err
}
storedKey := hash.Sum(nil)
mac = hmac.New(c.newHash, storedKey)
if _, err := mac.Write(c.authMsg.Bytes()); err != nil {
return nil, err
}
clientProof := mac.Sum(nil)
for i, b := range clientKey {
clientProof[i] ^= b
}
clientProof64 := make([]byte, b64.EncodedLen(len(clientProof)))
b64.Encode(clientProof64, clientProof)
return clientProof64, nil
}
func (c *Client) serverSignature() ([]byte, error) {
mac := hmac.New(c.newHash, c.saltedPass)
if _, err := mac.Write([]byte("Server Key")); err != nil {
return nil, err
}
serverKey := mac.Sum(nil)
mac = hmac.New(c.newHash, serverKey)
if _, err := mac.Write(c.authMsg.Bytes()); err != nil {
return nil, err
}
serverSignature := mac.Sum(nil)
encoded := make([]byte, b64.EncodedLen(len(serverSignature)))
b64.Encode(encoded, serverSignature)
return encoded, nil
}
gocbcore-10.2.3/searchcomponent.go 0000664 0000000 0000000 00000016712 14417540156 0017115 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"strings"
"time"
)
// SearchRowReader providers access to the rows of a view query
type SearchRowReader struct {
streamer *queryStreamer
}
// NextRow reads the next rows bytes from the stream
func (q *SearchRowReader) NextRow() []byte {
return q.streamer.NextRow()
}
// Err returns any errors that occurred during streaming.
func (q SearchRowReader) Err() error {
return q.streamer.Err()
}
// MetaData fetches the non-row bytes streamed in the response.
func (q *SearchRowReader) MetaData() ([]byte, error) {
return q.streamer.MetaData()
}
// Close immediately shuts down the connection
func (q *SearchRowReader) Close() error {
return q.streamer.Close()
}
// SearchQueryOptions represents the various options available for a search query.
type SearchQueryOptions struct {
IndexName string
Payload []byte
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
type jsonSearchErrorResponse struct {
Error string
}
func wrapSearchError(req *httpRequest, indexName string, query interface{}, err error, statusCode int) *SearchError {
if err == nil {
err = errors.New("search error")
}
ierr := &SearchError{
InnerError: err,
}
if req != nil {
ierr.Endpoint = req.Endpoint
ierr.RetryAttempts = req.RetryAttempts()
ierr.RetryReasons = req.RetryReasons()
}
ierr.HTTPResponseCode = statusCode
ierr.IndexName = indexName
ierr.Query = query
return ierr
}
func parseSearchError(req *httpRequest, indexName string, query interface{}, resp *HTTPResponse) *SearchError {
var err error
var errMsg string
respBody, readErr := ioutil.ReadAll(resp.Body)
if readErr == nil {
var respParse jsonSearchErrorResponse
parseErr := json.Unmarshal(respBody, &respParse)
if parseErr == nil {
errMsg = respParse.Error
}
}
if resp.StatusCode == 500 {
err = errInternalServerFailure
}
if resp.StatusCode == 401 || resp.StatusCode == 403 {
err = errAuthenticationFailure
}
if resp.StatusCode == 400 && strings.Contains(errMsg, "index not found") {
err = errIndexNotFound
}
if resp.StatusCode == 429 {
if strings.Contains(errMsg, "num_concurrent_requests") {
err = errRateLimitedFailure
} else if strings.Contains(errMsg, "num_queries_per_min") {
err = errRateLimitedFailure
} else if strings.Contains(errMsg, "ingress_mib_per_min") {
err = errRateLimitedFailure
} else if strings.Contains(errMsg, "egress_mib_per_min") {
err = errRateLimitedFailure
}
}
errOut := wrapSearchError(req, indexName, query, err, resp.StatusCode)
errOut.ErrorText = errMsg
return errOut
}
type searchQueryComponent struct {
httpComponent *httpComponent
tracer *tracerComponent
}
func newSearchQueryComponent(httpComponent *httpComponent, tracer *tracerComponent) *searchQueryComponent {
return &searchQueryComponent{
httpComponent: httpComponent,
tracer: tracer,
}
}
// SearchQuery executes a Search query
func (sqc *searchQueryComponent) SearchQuery(opts SearchQueryOptions, cb SearchQueryCallback) (PendingOp, error) {
tracer := sqc.tracer.StartTelemeteryHandler(metricValueServiceSearchValue, "SearchQuery", opts.TraceContext)
var payloadMap map[string]interface{}
err := json.Unmarshal(opts.Payload, &payloadMap)
if err != nil {
tracer.Finish()
return nil, wrapSearchError(nil, "", nil, wrapError(err, "expected a JSON payload"), 0)
}
var ctlMap map[string]interface{}
if foundCtlMap, ok := payloadMap["ctl"]; ok {
if coercedCtlMap, ok := foundCtlMap.(map[string]interface{}); ok {
ctlMap = coercedCtlMap
} else {
tracer.Finish()
return nil, wrapSearchError(nil, "", nil,
wrapError(errInvalidArgument, "expected ctl to be a map"), 0)
}
} else {
ctlMap = make(map[string]interface{})
}
indexName := opts.IndexName
query := payloadMap["query"]
ctx, cancel := context.WithCancel(context.Background())
reqURI := fmt.Sprintf("/api/index/%s/query", opts.IndexName)
ireq := &httpRequest{
Service: FtsService,
Method: "POST",
Path: reqURI,
Body: opts.Payload,
IsIdempotent: true,
Deadline: opts.Deadline,
RetryStrategy: opts.RetryStrategy,
RootTraceContext: tracer.RootContext(),
Context: ctx,
CancelFunc: cancel,
User: opts.User,
}
go func() {
res, err := sqc.searchQuery(ireq, indexName, query, payloadMap, ctlMap, tracer.StartTime())
if err != nil {
cancel()
tracer.Finish()
cb(nil, err)
return
}
tracer.Finish()
cb(res, nil)
}()
return ireq, nil
}
func (sqc *searchQueryComponent) searchQuery(ireq *httpRequest, indexName string, query interface{}, payloadMap map[string]interface{},
ctlMap map[string]interface{}, startTime time.Time) (*SearchRowReader, error) {
for {
{
if !ireq.Deadline.IsZero() {
// Produce an updated payload with the appropriate timeout
timeoutLeft := time.Until(ireq.Deadline)
if timeoutLeft <= 0 {
err := &TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "N1QLQuery",
Opaque: ireq.Identifier(),
TimeObserved: time.Since(startTime),
RetryReasons: ireq.retryReasons,
RetryAttempts: ireq.retryCount,
LastDispatchedTo: ireq.Endpoint,
}
return nil, wrapSearchError(nil, indexName, query, err, 0)
}
ctlMap["timeout"] = timeoutLeft / time.Millisecond
payloadMap["ctl"] = ctlMap
}
newPayload, err := json.Marshal(payloadMap)
if err != nil {
return nil, wrapSearchError(nil, indexName, query,
wrapError(err, "failed to produce payload"), 0)
}
ireq.Body = newPayload
}
resp, err := sqc.httpComponent.DoInternalHTTPRequest(ireq, false)
if err != nil {
if errors.Is(err, ErrRequestCanceled) {
return nil, err
}
// execHTTPRequest will handle retrying due to in-flight socket close based
// on whether or not IsIdempotent is set on the httpRequest
return nil, wrapSearchError(ireq, indexName, query, err, 0)
}
if resp.StatusCode != 200 {
searchErr := parseSearchError(ireq, indexName, query, resp)
var retryReason RetryReason
if searchErr.HTTPResponseCode == 429 && !errors.Is(searchErr, ErrRateLimitedFailure) {
retryReason = SearchTooManyRequestsRetryReason
}
if retryReason == nil {
// searchErr is already wrapped here
return nil, searchErr
}
shouldRetry, retryTime := retryOrchMaybeRetry(ireq, retryReason)
if !shouldRetry {
// searchErr is already wrapped here
return nil, searchErr
}
select {
case <-time.After(time.Until(retryTime)):
continue
case <-time.After(time.Until(ireq.Deadline)):
err := &TimeoutError{
InnerError: errUnambiguousTimeout,
OperationID: "SearchQuery",
Opaque: ireq.Identifier(),
TimeObserved: time.Since(startTime),
RetryReasons: ireq.retryReasons,
RetryAttempts: ireq.retryCount,
LastDispatchedTo: ireq.Endpoint,
}
return nil, wrapSearchError(ireq, indexName, query, err, 0)
}
}
streamer, err := newQueryStreamer(resp.Body, "hits")
if err != nil {
respBody, readErr := ioutil.ReadAll(resp.Body)
if readErr != nil {
logDebugf("Failed to read response body: %v", readErr)
}
sErr := wrapSearchError(ireq, indexName, query, err, resp.StatusCode)
sErr.ErrorText = string(respBody)
return nil, sErr
}
return &SearchRowReader{
streamer: streamer,
}, nil
}
}
gocbcore-10.2.3/searchcomponent_test.go 0000664 0000000 0000000 00000001657 14417540156 0020156 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"bytes"
"encoding/json"
"io/ioutil"
)
// TestSearchComponentNilRows tests the case where the server returns a rows field but it's set to a null value.
func (suite *UnitTestSuite) TestSearchComponentNilRows() {
d, err := suite.LoadRawTestDataset("search_hits_nil")
suite.Require().Nil(err)
qStreamer, err := newQueryStreamer(ioutil.NopCloser(bytes.NewBuffer(d)), "hits")
suite.Require().Nil(err, err)
reader := SearchRowReader{
streamer: qStreamer,
}
numRows := 0
for reader.NextRow() != nil {
numRows++
}
suite.Assert().Zero(numRows)
err = reader.Err()
suite.Require().Nil(err, err)
metaBytes, err := reader.MetaData()
suite.Require().Nil(err, err)
var meta map[string]interface{}
err = json.Unmarshal(metaBytes, &meta)
suite.Require().Nil(err, err)
status := meta["status"].(map[string]interface{})
errs := status["errors"].(map[string]interface{})
suite.Assert().Len(errs, 6)
}
gocbcore-10.2.3/seedcfgcontroller.go 0000664 0000000 0000000 00000002016 14417540156 0017421 0 ustar 00root root 0000000 0000000 package gocbcore
type seedConfigController struct {
*baseHTTPConfigController
seed string
iterNum uint64
}
func newSeedConfigController(seed, bucketName string, props httpPollerProperties,
cfgMgr *configManagementComponent) *seedConfigController {
scc := &seedConfigController{
seed: seed,
}
scc.baseHTTPConfigController = newBaseHTTPConfigController(bucketName, props, cfgMgr, scc.GetEndpoint)
return scc
}
func (scc *seedConfigController) GetEndpoint(iterNum uint64) string {
if scc.iterNum == iterNum {
return ""
}
scc.iterNum = iterNum
return scc.seed
}
// Pause was added solely for testing purposes and we don't need to do anything with it for this.
// Once we move to Gocaves for mocking then Pause will go away.
func (scc *seedConfigController) Pause(paused bool) {
}
func (scc *seedConfigController) Run() {
scc.DoLoop()
}
func (scc *seedConfigController) PollerError() error {
return scc.Error()
}
// We're already a http poller so do nothing
func (scc *seedConfigController) ForceHTTPPoller() {
}
gocbcore-10.2.3/statscomponent.go 0000664 0000000 0000000 00000014341 14417540156 0017002 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"sync"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
type statsComponent struct {
kvMux *kvMux
tracer *tracerComponent
defaultRetryStrategy RetryStrategy
}
func newStatsComponent(kvMux *kvMux, defaultRetry RetryStrategy, tracer *tracerComponent) *statsComponent {
return &statsComponent{
kvMux: kvMux,
tracer: tracer,
defaultRetryStrategy: defaultRetry,
}
}
func (sc *statsComponent) Stats(opts StatsOptions, cb StatsCallback) (PendingOp, error) {
tracer := sc.tracer.StartTelemeteryHandler(metricValueServiceKeyValue, "Stats", opts.TraceContext)
iter, err := sc.kvMux.PipelineSnapshot()
if err != nil {
tracer.Finish()
return nil, err
}
stats := make(map[string]SingleServerStats)
var statsLock sync.Mutex
op := new(multiPendingOp)
op.isIdempotent = true
var expected uint32
pipelines := make([]*memdPipeline, 0)
switch target := opts.Target.(type) {
case nil:
iter.Iterate(0, func(pipeline *memdPipeline) bool {
pipelines = append(pipelines, pipeline)
expected++
return false
})
case VBucketIDStatsTarget:
expected = 1
srvIdx, err := iter.NodeByVbucket(target.VbID, 0)
if err != nil {
return nil, err
}
pipelines = append(pipelines, iter.PipelineAt(srvIdx))
default:
return nil, errInvalidArgument
}
opHandledLocked := func() {
completed := op.IncrementCompletedOps()
if expected-completed == 0 {
tracer.Finish()
cb(&StatsResult{
Servers: stats,
}, nil)
}
}
var userFrame *memd.UserImpersonationFrame
if len(opts.User) > 0 {
userFrame = &memd.UserImpersonationFrame{
User: []byte(opts.User),
}
}
if opts.RetryStrategy == nil {
opts.RetryStrategy = sc.defaultRetryStrategy
}
for _, pipeline := range pipelines {
serverAddress := pipeline.Address()
handler := func(resp *memdQResponse, req *memdQRequest, err error) {
statsLock.Lock()
defer statsLock.Unlock()
// Fetch the specific stats key for this server. Creating a new entry
// for the server if we did not previously have one.
curStats, ok := stats[serverAddress]
if !ok {
stats[serverAddress] = SingleServerStats{
Stats: make(map[string]string),
}
curStats = stats[serverAddress]
}
if err != nil {
// Store the first (and hopefully only) error into the Error field of this
// server's stats entry.
if curStats.Error == nil {
curStats.Error = err
} else {
logDebugf("Got additional error for stats: %s: %v", serverAddress, err)
}
opHandledLocked()
return
}
// Check if the key and value length is zero. This indicates that we have reached
// the ending of the stats listing by this server.
if len(resp.Key) == 0 && len(resp.Value) == 0 {
// As this is a persistent request, we must manually cancel it to remove
// it from the pending ops list. To ensure we do not race multiple cancels,
// we only handle it as completed the one time cancellation succeeds.
if req.internalCancel(err) {
opHandledLocked()
}
return
}
curStats.StatsKeys = append(curStats.StatsKeys, resp.Key)
curStats.StatsChunks = append(curStats.StatsChunks, resp.Value)
if len(resp.Key) == 0 {
// We do this for the sake of consistency.
curStats.Stats[""] += string(resp.Value)
} else {
// Add the stat for this server to the list of stats.
curStats.Stats[string(resp.Key)] += string(resp.Value)
}
// If we don't reassign this then we lose any values added to StatsKeys and StatsChunks.
stats[serverAddress] = curStats
}
req := &memdQRequest{
Packet: memd.Packet{
Magic: memd.CmdMagicReq,
Command: memd.CmdStat,
Datatype: 0,
Cas: 0,
Key: []byte(opts.Key),
Value: nil,
UserImpersonationFrame: userFrame,
},
Persistent: true,
Callback: handler,
RootTraceContext: tracer.RootContext(),
RetryStrategy: opts.RetryStrategy,
}
curOp, err := sc.kvMux.DispatchDirectToAddress(req, pipeline)
if err != nil {
statsLock.Lock()
stats[serverAddress] = SingleServerStats{
Error: err,
}
opHandledLocked()
statsLock.Unlock()
continue
}
if !opts.Deadline.IsZero() {
start := time.Now()
req.SetTimer(time.AfterFunc(opts.Deadline.Sub(start), func() {
connInfo := req.ConnectionInfo()
count, reasons := req.Retries()
req.cancelWithCallbackAndFinishTracer(&TimeoutError{
InnerError: errAmbiguousTimeout,
OperationID: "Unlock",
Opaque: req.Identifier(),
TimeObserved: time.Since(start),
RetryReasons: reasons,
RetryAttempts: count,
LastDispatchedTo: connInfo.lastDispatchedTo,
LastDispatchedFrom: connInfo.lastDispatchedFrom,
LastConnectionID: connInfo.lastConnectionID,
}, tracer)
}))
}
op.ops = append(op.ops, curOp)
}
return op, nil
}
// SingleServerStats represents the stats returned from a single server.
type SingleServerStats struct {
Stats map[string]string
// StatsKeys and StatsChunks provide access to the raw keys and values returned on a per packet basis.
// This is useful for stats keys such as connections which, unlike most stats keys, return us a complex object
// per packet. Keys and chunks maintain the same ordering for indexes.
StatsKeys [][]byte
StatsChunks [][]byte
Error error
}
// StatsTarget is used for providing a specific target to the Stats operation.
type StatsTarget interface {
}
// VBucketIDStatsTarget indicates that a specific vbucket should be targeted by the Stats operation.
type VBucketIDStatsTarget struct {
VbID uint16
}
// StatsOptions encapsulates the parameters for a Stats operation.
type StatsOptions struct {
Key string
// Target indicates that something specific should be targeted by the operation. If left nil
// then the stats command will be sent to all servers.
Target StatsTarget
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
// StatsResult encapsulates the result of a Stats operation.
type StatsResult struct {
Servers map[string]SingleServerStats
}
gocbcore-10.2.3/testbenchsuite_test.go 0000664 0000000 0000000 00000011200 14417540156 0020000 0 ustar 00root root 0000000 0000000 package gocbcore
import (
cavescli "github.com/couchbaselabs/gocaves/client"
"log"
"sync"
"testing"
"time"
)
var (
globalBenchSuite *BenchTestSuite
globalBenchLock sync.Mutex
)
type BenchTestSuite struct {
*TestConfig
agent *Agent
mockInst *cavescli.Client
runID string
}
func GetBenchSuite() *BenchTestSuite {
globalBenchLock.Lock()
if globalBenchSuite == nil {
s := &BenchTestSuite{}
s.Setup()
globalBenchSuite = s
}
s := globalBenchSuite
globalBenchLock.Unlock()
return s
}
func (suite *BenchTestSuite) initAgent(config AgentConfig) (*Agent, error) {
agent, err := CreateAgent(&config)
if err != nil {
return nil, err
}
ch := make(chan error)
_, err = agent.WaitUntilReady(
time.Now().Add(10*time.Second),
WaitUntilReadyOptions{},
func(result *WaitUntilReadyResult, err error) {
ch <- err
},
)
if err != nil {
return nil, err
}
err = <-ch
if err != nil {
return nil, err
}
return agent, nil
}
func (suite *BenchTestSuite) Setup() {
if globalTestConfig.ConnStr == "" {
suite.mockInst, suite.runID = setupMock(true)
}
suite.TestConfig = globalTestConfig
config := makeAgentConfig(globalTestConfig)
config.IoConfig.UseCollections = suite.SupportsFeature(TestFeatureCollections)
config.BucketName = globalTestConfig.BucketName
var err error
suite.agent, err = suite.initAgent(config)
if err != nil {
panic(err)
}
}
func (suite *BenchTestSuite) Close() error {
if err := suite.agent.Close(); err != nil {
return err
}
if suite.mockInst != nil {
_, err := suite.mockInst.EndTesting(suite.runID)
if err != nil {
log.Printf("Failed to end testing: %v", err)
return err
}
err = suite.mockInst.Shutdown()
if err != nil {
return err
}
}
return nil
}
func (suite *BenchTestSuite) GetHarness(b *testing.B) *TestSubHarness {
return makeTestSubHarness(b)
}
func (suite *BenchTestSuite) GetAgent() *Agent {
return suite.agent
}
func (suite *BenchTestSuite) GetAgentAndHarness(b *testing.B) (*Agent, *TestSubHarness) {
h := suite.GetHarness(b)
return suite.GetAgent(), h
}
func (suite *BenchTestSuite) SupportsFeature(feature TestFeatureCode) bool {
featureFlagValue := 0
for _, featureFlag := range suite.FeatureFlags {
if featureFlag.Feature == feature || featureFlag.Feature == "*" {
if featureFlag.Enabled {
featureFlagValue = +1
} else {
featureFlagValue = -1
}
}
}
if featureFlagValue == -1 {
return false
} else if featureFlagValue == +1 {
return true
}
switch feature {
case TestFeatureCollections:
return !suite.ClusterVersion.Lower(srvVer700)
case TestFeatureTransactions:
return !suite.ClusterVersion.Lower(srvVer700)
}
panic("found unsupported feature code")
}
func (suite *BenchTestSuite) EnsureSupportsFeature(feature TestFeatureCode, b *testing.B) {
if !suite.SupportsFeature(feature) {
b.Skipf("Skipping test due to disabled feature code: %s", feature)
}
}
func (suite *BenchTestSuite) RunParallel(b *testing.B, runCB func(func(error)) error) {
agent := suite.GetAgent()
maxConcurrency := agent.kvMux.queueSize
buf := make(chan struct{}, maxConcurrency)
for i := 0; i < maxConcurrency; i++ {
buf <- struct{}{}
}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
var wg sync.WaitGroup
for pb.Next() {
<-buf
wg.Add(1)
err := runCB(func(cbErr error) {
if cbErr != nil {
b.Errorf("Operation failed: %v", cbErr)
}
wg.Done()
buf <- struct{}{}
})
if err != nil {
buf <- struct{}{}
wg.Done()
b.Fatalf("Failed to run operation: %v", err)
}
}
wg.Wait()
})
}
func (suite *BenchTestSuite) RunParallelTxn(b *testing.B, config *TransactionsConfig, runCB func(*Transaction, func(error)) error) {
agent := suite.GetAgent()
maxConcurrency := agent.kvMux.queueSize
buf := make(chan struct{}, maxConcurrency)
for i := 0; i < maxConcurrency; i++ {
buf <- struct{}{}
}
txns, err := InitTransactions(config)
if err != nil {
b.Fatalf("Failed to init transactions: %v", err)
}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
txn, err := txns.BeginTransaction(nil)
if err != nil {
b.Fatalf("Failed to begin transactios: %v", err)
}
err = txn.NewAttempt()
if err != nil {
b.Fatalf("Failed to create new attempt: %v", err)
}
var wg sync.WaitGroup
for pb.Next() {
<-buf
wg.Add(1)
err := runCB(txn, func(cbErr error) {
if cbErr != nil {
b.Errorf("Operation failed: %v", cbErr)
}
wg.Done()
buf <- struct{}{}
})
if err != nil {
buf <- struct{}{}
wg.Done()
b.Fatalf("Failed to run operation: %v", err)
}
}
wg.Wait()
err = testBlkCommit(txn)
if err != nil {
b.Fatalf("Failed to commit attempt: %v", err)
}
})
}
gocbcore-10.2.3/testconfig_test.go 0000664 0000000 0000000 00000002230 14417540156 0017117 0 ustar 00root root 0000000 0000000 package gocbcore
import "crypto/x509"
var globalTestConfig *TestConfig
type TestConfig struct {
ConnStr string
BucketName string
MemdBucketName string
ScopeName string
CollectionName string
Authenticator AuthProvider
CAProvider func() *x509.CertPool
ClusterVersion *NodeVersion
FeatureFlags []TestFeatureFlag
MockPath string
}
func (tc *TestConfig) Clone() *TestConfig {
return &TestConfig{
ConnStr: tc.ConnStr,
BucketName: tc.BucketName,
MemdBucketName: tc.MemdBucketName,
ScopeName: tc.ScopeName,
CollectionName: tc.CollectionName,
Authenticator: tc.Authenticator,
CAProvider: tc.CAProvider,
ClusterVersion: tc.ClusterVersion,
FeatureFlags: tc.FeatureFlags,
MockPath: tc.MockPath,
}
}
var globalDCPTestConfig *DCPTestConfig
type DCPTestConfig struct {
ConnStr string
BucketName string
Scope uint32
Collections []uint32
Authenticator AuthProvider
CAProvider func() *x509.CertPool
ClusterVersion *NodeVersion
FeatureFlags []TestFeatureFlag
NumMutations int
NumDeletions int
NumExpirations int
NumScopes int
NumCollections int
}
gocbcore-10.2.3/testdata/ 0000775 0000000 0000000 00000000000 14417540156 0015200 5 ustar 00root root 0000000 0000000 gocbcore-10.2.3/testdata/bucket_config_with_external_addresses.json 0000664 0000000 0000000 00000146272 14417540156 0025703 0 ustar 00root root 0000000 0000000 {
"rev":1073,
"name":"default",
"uri":"/pools/default/buckets/default?bucket_uuid=ee7160b1f5392bcdbfc085c98b460999",
"streamingUri":"/pools/default/bucketsStreaming/default?bucket_uuid=ee7160b1f5392bcdbfc085c98b460999",
"nodes":[
{
"couchApiBase":"http://172.17.0.2:8092/default%2Bee7160b1f5392bcdbfc085c98b460999",
"hostname":"172.17.0.2:8091",
"ports":{
"proxy":11211,
"direct":11210
},
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234",
"ports":{
"mgmt":32790,
"kv":32775
}
}
}
},
{
"couchApiBase":"http://172.17.0.3:8092/default%2Bee7160b1f5392bcdbfc085c98b460999",
"hostname":"172.17.0.3:8091",
"ports":{
"proxy":11211,
"direct":11210
},
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234",
"ports":{
"mgmt":32814,
"kv":32799
}
}
}
},
{
"couchApiBase":"http://172.17.0.4:8092/default%2Bee7160b1f5392bcdbfc085c98b460999",
"hostname":"172.17.0.4:8091",
"ports":{
"proxy":11211,
"direct":11210
},
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234",
"ports":{
"mgmt":32838,
"kv":32823
}
}
}
}
],
"nodesExt":[
{
"services":{
"mgmt":8091,
"mgmtSSL":18091,
"fts":8094,
"ftsSSL":18094,
"indexAdmin":9100,
"indexScan":9101,
"indexHttp":9102,
"indexStreamInit":9103,
"indexStreamCatchup":9104,
"indexStreamMaint":9105,
"indexHttps":19102,
"capiSSL":18092,
"capi":8092,
"kvSSL":11207,
"projector":9999,
"kv":11210,
"moxi":11211,
"n1ql":8093,
"n1qlSSL":18093
},
"thisNode":true,
"hostname":"172.17.0.2",
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234",
"ports":{
"mgmt":32790,
"mgmtSSL":32773,
"fts":32787,
"ftsSSL":32770,
"kv":32775,
"kvSSL":32776,
"capi":32789,
"capiSSL":32772,
"n1ql":32788,
"n1qlSSL":32771
}
}
}
},
{
"services":{
"mgmt":8091,
"mgmtSSL":18091,
"fts":8094,
"ftsSSL":18094,
"indexAdmin":9100,
"indexScan":9101,
"indexHttp":9102,
"indexStreamInit":9103,
"indexStreamCatchup":9104,
"indexStreamMaint":9105,
"indexHttps":19102,
"capiSSL":18092,
"capi":8092,
"kvSSL":11207,
"projector":9999,
"kv":11210,
"moxi":11211,
"n1ql":8093,
"n1qlSSL":18093
},
"hostname":"172.17.0.3",
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234",
"ports":{
"mgmt":32814,
"mgmtSSL":32797,
"fts":32811,
"ftsSSL":32794,
"kv":32799,
"kvSSL":32800,
"capi":32813,
"capiSSL":32796,
"n1ql":32812,
"n1qlSSL":32795
}
}
}
},
{
"services":{
"mgmt":8091,
"mgmtSSL":18091,
"fts":8094,
"ftsSSL":18094,
"indexAdmin":9100,
"indexScan":9101,
"indexHttp":9102,
"indexStreamInit":9103,
"indexStreamCatchup":9104,
"indexStreamMaint":9105,
"indexHttps":19102,
"capiSSL":18092,
"capi":8092,
"kvSSL":11207,
"projector":9999,
"kv":11210,
"moxi":11211,
"n1ql":8093,
"n1qlSSL":18093
},
"hostname":"172.17.0.4",
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234",
"ports":{
"mgmt":32838,
"mgmtSSL":32821,
"fts":32835,
"ftsSSL":32818,
"kv":32823,
"kvSSL":32824,
"capi":32837,
"capiSSL":32820,
"n1ql":32836,
"n1qlSSL":32819
}
}
}
}
],
"nodeLocator":"vbucket",
"uuid":"ee7160b1f5392bcdbfc085c98b460999",
"ddocs":{
"uri":"/pools/default/buckets/default/ddocs"
},
"vBucketServerMap":{
"hashAlgorithm":"CRC",
"numReplicas":1,
"serverList":[
"172.17.0.2:11210",
"172.17.0.3:11210",
"172.17.0.4:11210"
],
"vBucketMap":[
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
]
]
},
"bucketCapabilitiesVer":"",
"bucketCapabilities":[
"couchapi",
"xattr",
"dcp",
"cbhello",
"touch",
"cccp",
"xdcrCheckpointing",
"nodesExt"
]
}
gocbcore-10.2.3/testdata/bucket_config_with_external_addresses_without_ports.json 0000664 0000000 0000000 00000143733 14417540156 0030714 0 ustar 00root root 0000000 0000000 {
"rev":1073,
"name":"default",
"uri":"/pools/default/buckets/default?bucket_uuid=ee7160b1f5392bcdbfc085c98b460999",
"streamingUri":"/pools/default/bucketsStreaming/default?bucket_uuid=ee7160b1f5392bcdbfc085c98b460999",
"nodes":[
{
"couchApiBase":"http://172.17.0.2:8092/default%2Bee7160b1f5392bcdbfc085c98b460999",
"hostname":"172.17.0.2:8091",
"ports":{
"proxy":11211,
"direct":11210
},
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234"
}
}
},
{
"couchApiBase":"http://172.17.0.3:8092/default%2Bee7160b1f5392bcdbfc085c98b460999",
"hostname":"172.17.0.3:8091",
"ports":{
"proxy":11211,
"direct":11210
},
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234"
}
}
},
{
"couchApiBase":"http://172.17.0.4:8092/default%2Bee7160b1f5392bcdbfc085c98b460999",
"hostname":"172.17.0.4:8091",
"ports":{
"proxy":11211,
"direct":11210
},
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234"
}
}
}
],
"nodesExt":[
{
"services":{
"mgmt":8091,
"mgmtSSL":18091,
"fts":8094,
"ftsSSL":18094,
"indexAdmin":9100,
"indexScan":9101,
"indexHttp":9102,
"indexStreamInit":9103,
"indexStreamCatchup":9104,
"indexStreamMaint":9105,
"indexHttps":19102,
"capiSSL":18092,
"capi":8092,
"kvSSL":11207,
"projector":9999,
"kv":11210,
"moxi":11211,
"n1ql":8093,
"n1qlSSL":18093
},
"thisNode":true,
"hostname":"172.17.0.2",
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234"
}
}
},
{
"services":{
"mgmt":8091,
"mgmtSSL":18091,
"fts":8094,
"ftsSSL":18094,
"indexAdmin":9100,
"indexScan":9101,
"indexHttp":9102,
"indexStreamInit":9103,
"indexStreamCatchup":9104,
"indexStreamMaint":9105,
"indexHttps":19102,
"capiSSL":18092,
"capi":8092,
"kvSSL":11207,
"projector":9999,
"kv":11210,
"moxi":11211,
"n1ql":8093,
"n1qlSSL":18093
},
"hostname":"172.17.0.3",
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234"
}
}
},
{
"services":{
"mgmt":8091,
"mgmtSSL":18091,
"fts":8094,
"ftsSSL":18094,
"indexAdmin":9100,
"indexScan":9101,
"indexHttp":9102,
"indexStreamInit":9103,
"indexStreamCatchup":9104,
"indexStreamMaint":9105,
"indexHttps":19102,
"capiSSL":18092,
"capi":8092,
"kvSSL":11207,
"projector":9999,
"kv":11210,
"moxi":11211,
"n1ql":8093,
"n1qlSSL":18093
},
"hostname":"172.17.0.4",
"alternateAddresses":{
"external":{
"hostname":"192.168.132.234"
}
}
}
],
"nodeLocator":"vbucket",
"uuid":"ee7160b1f5392bcdbfc085c98b460999",
"ddocs":{
"uri":"/pools/default/buckets/default/ddocs"
},
"vBucketServerMap":{
"hashAlgorithm":"CRC",
"numReplicas":1,
"serverList":[
"172.17.0.2:11210",
"172.17.0.3:11210",
"172.17.0.4:11210"
],
"vBucketMap":[
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
1
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
0,
2
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
0
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
1,
2
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
0
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
],
[
2,
1
]
]
},
"bucketCapabilitiesVer":"",
"bucketCapabilities":[
"couchapi",
"xattr",
"dcp",
"cbhello",
"touch",
"cccp",
"xdcrCheckpointing",
"nodesExt"
]
}
gocbcore-10.2.3/testdata/bucket_config_with_rev_epoch.json 0000664 0000000 0000000 00000122004 14417540156 0023761 0 ustar 00root root 0000000 0000000 {
"rev": 2,
"revEpoch": 2,
"name": "travel-sample",
"nodeLocator": "vbucket",
"uuid": "ae4c45a9818aee3bd83bf65d10c99d9d",
"uri": "/pools/default/buckets/travel-sample?bucket_uuid=ae4c45a9818aee3bd83bf65d10c99d9d",
"streamingUri": "/pools/default/bucketsStreaming/travel-sample?bucket_uuid=ae4c45a9818aee3bd83bf65d10c99d9d",
"bucketCapabilitiesVer": "",
"bucketCapabilities": [
"collections",
"durableWrite",
"tombstonedUserXAttrs",
"couchapi",
"subdoc.ReplaceBodyWithXattr",
"subdoc.DocumentMacroSupport",
"dcp",
"cbhello",
"touch",
"cccp",
"xdcrCheckpointing",
"nodesExt",
"xattr"
],
"collectionsManifestUid": "1",
"ddocs": {
"uri": "/pools/default/buckets/travel-sample/ddocs"
},
"vBucketServerMap": {
"hashAlgorithm": "CRC",
"numReplicas": 1,
"serverList": [
"$HOST:11210"
],
"vBucketMap": [
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
],
[
0,
-1
]
]
},
"nodes": [
{
"couchApiBase": "http://$HOST:8092/travel-sample%2Bae4c45a9818aee3bd83bf65d10c99d9d",
"hostname": "$HOST:8091",
"ports": {
"direct": 11210
}
}
],
"nodesExt": [
{
"services": {
"backupAPI": 8097,
"backupAPIHTTPS": 18097,
"backupGRPC": 9124,
"capi": 8092,
"capiSSL": 18092,
"eventingAdminPort": 8096,
"eventingDebug": 9140,
"eventingSSL": 18096,
"fts": 8094,
"ftsGRPC": 9130,
"ftsGRPCSSL": 19130,
"ftsSSL": 18094,
"indexAdmin": 9100,
"indexHttp": 9102,
"indexHttps": 19102,
"indexScan": 9101,
"indexStreamCatchup": 9104,
"indexStreamInit": 9103,
"indexStreamMaint": 9105,
"kv": 11210,
"kvSSL": 11207,
"mgmt": 8091,
"mgmtSSL": 18091,
"n1ql": 8093,
"n1qlSSL": 18093,
"projector": 9999
},
"thisNode": true
}
],
"clusterCapabilitiesVer": [
1,
0
],
"clusterCapabilities": {
"n1ql": [
"enhancedPreparedStatements"
]
}
}
gocbcore-10.2.3/testdata/err_map70_v1.json 0000664 0000000 0000000 00000024207 14417540156 0020302 0 ustar 00root root 0000000 0000000 {
"version": 1,
"revision": 2,
"errors": {
"0": {
"name": "SUCCESS",
"desc": "Success",
"attrs": [
"success"
]
},
"1": {
"name": "KEY_ENOENT",
"desc": "Not Found",
"attrs": [
"item-only"
]
},
"2": {
"name": "KEY_EEXISTS",
"desc": "key already exists, or CAS mismatch",
"attrs": [
"item-only"
]
},
"3": {
"name": "E2BIG",
"desc": "Value is too big",
"attrs": [
"item-only",
"invalid-input"
]
},
"4": {
"name": "EINVAL",
"desc": "Invalid packet",
"attrs": [
"internal",
"invalid-input"
]
},
"5": {
"name": "NOT_STORED",
"desc": "Not Stored",
"attrs": [
"internal",
"item-only"
]
},
"6": {
"name": "DELTA_BADVAL",
"desc": "Existing document not a number",
"attrs": [
"item-only",
"invalid-input"
]
},
"7": {
"name": "NOT_MY_VBUCKET",
"desc": "Server does not know about this vBucket",
"attrs": [
"fetch-config",
"invalid-input"
]
},
"8": {
"name": "NO_BUCKET",
"desc": "Not connected to any bucket",
"attrs": [
"conn-state-invalidated"
]
},
"9": {
"name": "LOCKED",
"desc": "Requested resource is locked",
"attrs": [
"item-locked",
"item-only",
"retry-now"
]
},
"1f": {
"name": "AUTH_STALE",
"desc": "Reauthentication required",
"attrs": [
"conn-state-invalidated",
"auth"
]
},
"20": {
"name": "AUTH_ERROR",
"desc": "Authentication failed",
"attrs": [
"conn-state-invalidated",
"auth"
]
},
"21": {
"name": "AUTH_CONTINUE",
"desc": "Continue authentication processs",
"attrs": [
"special-handling"
]
},
"22": {
"name": "ERANGE",
"desc": "Invalid range requested",
"attrs": [
"invalid-input"
]
},
"23": {
"name": "ROLLBACK",
"desc": "Rollback",
"attrs": [
"dcp",
"special-handling"
]
},
"24": {
"name": "EACCESS",
"desc": "Not authorized for command",
"attrs": [
"support"
]
},
"25": {
"name": "NOT_INITIALIZED",
"desc": "Server not initialized",
"attrs": [
"conn-state-invalidated"
]
},
"80": {
"name": "UNKNOWN_FRAME_INFO",
"desc": "Unknown frame info identifier encountered. Maybe a newer server version knows about it",
"attrs": [
"support"
]
},
"81": {
"name": "UNKNOWN_COMMAND",
"desc": "Unknown command. Maybe a newer server version knows about it",
"attrs": [
"support"
]
},
"82": {
"name": "ENOMEM",
"desc": "No memory available to store item. Add memory or remove some items and try later",
"attrs": [
"temp",
"retry-later"
]
},
"83": {
"name": "NOT_SUPPORTED",
"desc": "Command not supported with current bucket type/configuration",
"attrs": [
"support"
]
},
"84": {
"name": "EINTERNAL",
"desc": "Internal error. Reconnect recommended",
"attrs": [
"internal",
"conn-state-invalidated"
]
},
"85": {
"name": "EBUSY",
"desc": "Busy, try again",
"attrs": [
"temp",
"retry-now"
]
},
"86": {
"name": "ETMPFAIL",
"desc": "Temporary failure. Try again",
"attrs": [
"temp",
"retry-now"
]
},
"87": {
"name": "XATTR_EINVAL",
"desc": "Invalid extended attribute",
"attrs": [
"invalid-input"
]
},
"88": {
"name": "UNKNOWN_COLLECTION",
"desc": "Operation specified an unknown collection.",
"attrs": [
"invalid-input"
]
},
"89": {
"name": "NO_COLLECTIONS_MANIFEST",
"desc": "No collections manifest has been set.",
"attrs": [
"retry-later"
]
},
"8a": {
"name": "CANNOT_APPLY_COLLECTIONS_MANIFEST",
"desc": "The manifest cannot applied to the bucket's vbuckets.",
"attrs": [
"invalid-input"
]
},
"8b": {
"name": "COLLECTIONS_MANIFEST_IS_AHEAD",
"desc": "The specified collection's manifest uid is greater than the requested vbucket's.",
"attrs": [
"retry-later"
]
},
"8c": {
"name": "UNKNOWN_SCOPE",
"desc": "Operation specified an unknown scope.",
"attrs": [
"invalid-input"
]
},
"8d": {
"name": "DCP stream-ID invalid",
"desc": "Operations stream-ID usage is incorrect.",
"attrs": [
"invalid-input"
]
},
"a0": {
"name": "DurabilityInvalidLevel",
"desc": "Durability level is invalid",
"attrs": [
"invalid-input"
]
},
"a1": {
"name": "DurabilityImpossible",
"desc": "Durability requirements are impossible to achieve",
"attrs": [
"item-only",
"retry-later"
]
},
"a2": {
"name": "SyncWriteInProgress",
"desc": "The requested key has a pending synchronous write",
"attrs": [
"item-only",
"retry-later"
]
},
"a3": {
"name": "SyncWriteAmbiguous",
"desc": "The SyncWrite request has not completed in the specified time and has ambiguous result - it may Succeed or Fail; but the final value is not yet known",
"attrs": [
"item-only"
]
},
"a4": {
"name": "SyncWriteReCommitInProgress",
"desc": "The requested key has a SyncWrite which is being re-committed.",
"attrs": [
"item-only",
"retry-later"
]
},
"c0": {
"name": "SUBDOC_PATH_ENOENT",
"desc": "Subdoc: Path not found in document",
"attrs": [
"subdoc",
"item-only"
]
},
"c1": {
"name": "SUBDOC_PATH_MISMATCH",
"desc": "Subdoc: Path and document disagree on structure",
"attrs": [
"subdoc",
"item-only"
]
},
"c2": {
"name": "SUBDOC_PATH_EINVAL",
"desc": "Subdoc: Invalid path (bad syntax or unacceptable semantics for command",
"attrs": [
"subdoc",
"invalid-input"
]
},
"c3": {
"name": "SUBDOC_PATH_E2BIG",
"desc": "Subdoc: Path size exceeds limit",
"attrs": [
"subdoc",
"invalid-input"
]
},
"c4": {
"name": "SUBDOC_PATH_E2DEEP",
"desc": "Subdoc: Path is too deep to be parsed",
"attrs": [
"subdoc",
"invalid-input"
]
},
"c5": {
"name": "SUBDOC_VALUE_CANTINSERT",
"desc": "Subdoc: Value invalid for insertion",
"attrs": [
"subdoc",
"invalid-input"
]
},
"c6": {
"name": "SUBDOC_DOC_NOTJSON",
"desc": "Subdoc: Document not JSON",
"attrs": [
"subdoc",
"item-only"
]
},
"c7": {
"name": "SUBDOC_NUM_ERANGE",
"desc": "Subdoc: Existing numeric value is not within range",
"attrs": [
"subdoc",
"item-only"
]
},
"c8": {
"name": "SUBDOC_DELTA_EINVAL",
"desc": "Subdoc: Invalid value passed for delta (out of range, or not an integer",
"attrs": [
"subdoc",
"item-only"
]
},
"c9": {
"name": "SUBDOC_PATH_EEXISTS",
"desc": "Subdoc: Path already exists",
"attrs": [
"subdoc",
"item-only"
]
},
"ca": {
"name": "SUBDOC_VALUE_ETOODEEP",
"desc": "Subdoc: Value is too deep, or would make the document too deep",
"attrs": [
"subdoc",
"invalid-input",
"item-only"
]
},
"cb": {
"name": "SUBDOC_INVALID_COMBO",
"desc": "Subdoc: Lookup and mutation commands found within single packet",
"attrs": [
"subdoc",
"invalid-input"
]
},
"cc": {
"name": "SUBDOC_MULTI_PATH_FAILURE",
"desc": "Subdoc: Some (or all) commands failed. Inspect payload for details",
"attrs": [
"subdoc",
"special-handling"
]
},
"cd": {
"name": "SUBDOC_SUCCESS_DELETED",
"desc": "Subdoc: Success, but the affected document was (and still is) deleted",
"attrs": [
"item-deleted",
"success",
"subdoc"
]
},
"ce": {
"name": "SUBDOC_XATTR_INVALID_FLAG_COMBO",
"desc": "Subdoc: The flag combination doesn't make any sense",
"attrs": [
"subdoc",
"invalid-input"
]
},
"cf": {
"name": "SUBDOC_XATTR_INVALID_KEY_COMBO",
"desc": "Subdoc: The key combination of the xattrs is not allowed",
"attrs": [
"subdoc",
"invalid-input"
]
},
"d0": {
"name": "SUBDOC_XATTR_UNKNOWN_MACRO",
"desc": "Subdoc: The server don't know about the specified macro",
"attrs": [
"subdoc",
"invalid-input"
]
},
"d1": {
"name": "SUBDOC_XATTR_UNKNOWN_VATTR",
"desc": "Subdoc: The server don't know about the specified virtual attribute",
"attrs": [
"subdoc",
"invalid-input"
]
},
"d2": {
"name": "SUBDOC_XATTR_CANT_MODIFY_VATTR",
"desc": "Subdoc: Can't modify virtual attributes",
"attrs": [
"subdoc",
"invalid-input"
]
},
"d3": {
"name": "SUBDOC_MULTI_PATH_FAILURE_DELETED",
"desc": "Subdoc: One or more paths in a multi-path command failed on a deleted document",
"attrs": [
"item-deleted",
"subdoc",
"special-handling"
]
},
"d4": {
"name": "SUBDOC_INVALID_XATTR_ORDER",
"desc": "Subdoc: Invalid XATTR order (xattrs should come first)",
"attrs": [
"subdoc",
"invalid-input"
]
},
"d5": {
"name": "SUBDOC_XATTR_UNKNOWN_VATTR_MACRO",
"desc": "Subdoc: The server don't know about (or support) the specified virtual macro",
"attrs": [
"subdoc",
"invalid-input"
]
}
}
}
gocbcore-10.2.3/testdata/err_map71_v2.json 0000664 0000000 0000000 00000026566 14417540156 0020316 0 ustar 00root root 0000000 0000000 {
"errors": {
"0": {
"attrs": [
"success"
],
"desc": "Success",
"name": "SUCCESS"
},
"1": {
"attrs": [
"item-only"
],
"desc": "Not Found",
"name": "KEY_ENOENT"
},
"2": {
"attrs": [
"item-only"
],
"desc": "key already exists, or CAS mismatch",
"name": "KEY_EEXISTS"
},
"3": {
"attrs": [
"item-only",
"invalid-input"
],
"desc": "Value is too big",
"name": "E2BIG"
},
"4": {
"attrs": [
"internal",
"invalid-input"
],
"desc": "Invalid packet",
"name": "EINVAL"
},
"5": {
"attrs": [
"internal",
"item-only"
],
"desc": "Not Stored",
"name": "NOT_STORED"
},
"6": {
"attrs": [
"item-only",
"invalid-input"
],
"desc": "Existing document not a number",
"name": "DELTA_BADVAL"
},
"7": {
"attrs": [
"fetch-config",
"invalid-input"
],
"desc": "Server does not know about this vBucket",
"name": "NOT_MY_VBUCKET"
},
"8": {
"attrs": [
"conn-state-invalidated"
],
"desc": "Not connected to any bucket",
"name": "NO_BUCKET"
},
"9": {
"attrs": [
"item-locked",
"item-only",
"retry-now"
],
"desc": "Requested resource is locked",
"name": "LOCKED"
},
"20": {
"attrs": [
"conn-state-invalidated",
"auth"
],
"desc": "Authentication failed",
"name": "AUTH_ERROR"
},
"21": {
"attrs": [
"special-handling"
],
"desc": "Continue authentication processs",
"name": "AUTH_CONTINUE"
},
"22": {
"attrs": [
"invalid-input"
],
"desc": "Invalid range requested",
"name": "ERANGE"
},
"23": {
"attrs": [
"dcp",
"special-handling"
],
"desc": "Rollback",
"name": "ROLLBACK"
},
"24": {
"attrs": [
"support"
],
"desc": "Not authorized for command",
"name": "EACCESS"
},
"25": {
"attrs": [
"conn-state-invalidated"
],
"desc": "Server not initialized",
"name": "NOT_INITIALIZED"
},
"30": {
"attrs": [
"temp",
"retry-later",
"rate-limit"
],
"desc": "Rate limited: Network Ingress",
"name": "RATE_LIMITED_NETWORK_INGRESS"
},
"31": {
"attrs": [
"temp",
"retry-later",
"rate-limit"
],
"desc": "Rate limited: Network Egress",
"name": "RATE_LIMITED_NETWORK_EGRESS"
},
"32": {
"attrs": [
"conn-state-invalidated",
"rate-limit"
],
"desc": "Rate limited: Max Connections",
"name": "RATE_LIMITED_MAX_CONNECTIONS"
},
"33": {
"attrs": [
"temp",
"retry-later",
"rate-limit"
],
"desc": "Rate limited: Max Commands",
"name": "RATE_LIMITED_MAX_COMMANDS"
},
"80": {
"attrs": [
"support"
],
"desc": "Unknown frame info identifier encountered. Maybe a newer server version knows about it",
"name": "UNKNOWN_FRAME_INFO"
},
"81": {
"attrs": [
"support"
],
"desc": "Unknown command. Maybe a newer server version knows about it",
"name": "UNKNOWN_COMMAND"
},
"82": {
"attrs": [
"temp",
"retry-later"
],
"desc": "No memory available to store item. Add memory or remove some items and try later",
"name": "ENOMEM"
},
"83": {
"attrs": [
"support"
],
"desc": "Command not supported with current bucket type/configuration",
"name": "NOT_SUPPORTED"
},
"84": {
"attrs": [
"internal",
"conn-state-invalidated"
],
"desc": "Internal error. Reconnect recommended",
"name": "EINTERNAL"
},
"85": {
"attrs": [
"temp",
"retry-now"
],
"desc": "Busy, try again",
"name": "EBUSY"
},
"86": {
"attrs": [
"temp",
"retry-now"
],
"desc": "Temporary failure. Try again",
"name": "ETMPFAIL"
},
"87": {
"attrs": [
"invalid-input"
],
"desc": "Invalid extended attribute",
"name": "XATTR_EINVAL"
},
"88": {
"attrs": [
"invalid-input"
],
"desc": "Operation specified an unknown collection.",
"name": "UNKNOWN_COLLECTION"
},
"89": {
"attrs": [
"retry-later"
],
"desc": "No collections manifest has been set.",
"name": "NO_COLLECTIONS_MANIFEST"
},
"1f": {
"attrs": [
"conn-state-invalidated",
"auth"
],
"desc": "Reauthentication required",
"name": "AUTH_STALE"
},
"8a": {
"attrs": [
"invalid-input"
],
"desc": "The manifest cannot applied to the bucket's vbuckets.",
"name": "CANNOT_APPLY_COLLECTIONS_MANIFEST"
},
"8c": {
"attrs": [
"invalid-input"
],
"desc": "Operation specified an unknown scope.",
"name": "UNKNOWN_SCOPE"
},
"8d": {
"attrs": [
"invalid-input"
],
"desc": "Operations stream-ID usage is incorrect.",
"name": "DCP stream-ID invalid"
},
"a": {
"attrs": [
"conn-state-invalidated"
],
"desc": "Stream not found",
"name": "STREAM_NOT_FOUND"
},
"a0": {
"attrs": [
"invalid-input"
],
"desc": "Durability level is invalid",
"name": "DurabilityInvalidLevel"
},
"a1": {
"attrs": [
"item-only",
"invalid-input"
],
"desc": "Durability requirements are impossible to achieve",
"name": "DurabilityImpossible"
},
"a2": {
"attrs": [
"item-only",
"retry-later"
],
"desc": "The requested key has a pending synchronous write",
"name": "SyncWriteInProgress"
},
"a3": {
"attrs": [
"item-only"
],
"desc": "The SyncWrite request has not completed in the specified time and has ambiguous result - it may Succeed or Fail; but the final value is not yet known",
"name": "SyncWriteAmbiguous"
},
"a4": {
"attrs": [
"item-only",
"retry-later"
],
"desc": "The requested key has a SyncWrite which is being re-committed.",
"name": "SyncWriteReCommitInProgress"
},
"b": {
"attrs": [
"conn-state-invalidated"
],
"desc": "Opaque does not match",
"name": "OPAQUE_NO_MATCH"
},
"c0": {
"attrs": [
"subdoc",
"item-only"
],
"desc": "Subdoc: Path not found in document",
"name": "SUBDOC_PATH_ENOENT"
},
"c1": {
"attrs": [
"subdoc",
"item-only"
],
"desc": "Subdoc: Path and document disagree on structure",
"name": "SUBDOC_PATH_MISMATCH"
},
"c2": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: Invalid path (bad syntax or unacceptable semantics for command",
"name": "SUBDOC_PATH_EINVAL"
},
"c3": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: Path size exceeds limit",
"name": "SUBDOC_PATH_E2BIG"
},
"c4": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: Path is too deep to be parsed",
"name": "SUBDOC_PATH_E2DEEP"
},
"c5": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: Value invalid for insertion",
"name": "SUBDOC_VALUE_CANTINSERT"
},
"c6": {
"attrs": [
"subdoc",
"item-only"
],
"desc": "Subdoc: Document not JSON",
"name": "SUBDOC_DOC_NOTJSON"
},
"c7": {
"attrs": [
"subdoc",
"item-only"
],
"desc": "Subdoc: Existing numeric value is not within range",
"name": "SUBDOC_NUM_ERANGE"
},
"c8": {
"attrs": [
"subdoc",
"item-only"
],
"desc": "Subdoc: Invalid value passed for delta (out of range, or not an integer",
"name": "SUBDOC_DELTA_EINVAL"
},
"c9": {
"attrs": [
"subdoc",
"item-only"
],
"desc": "Subdoc: Path already exists",
"name": "SUBDOC_PATH_EEXISTS"
},
"ca": {
"attrs": [
"subdoc",
"invalid-input",
"item-only"
],
"desc": "Subdoc: Value is too deep, or would make the document too deep",
"name": "SUBDOC_VALUE_ETOODEEP"
},
"cb": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: Lookup and mutation commands found within single packet",
"name": "SUBDOC_INVALID_COMBO"
},
"cc": {
"attrs": [
"subdoc",
"special-handling"
],
"desc": "Subdoc: Some (or all) commands failed. Inspect payload for details",
"name": "SUBDOC_MULTI_PATH_FAILURE"
},
"cd": {
"attrs": [
"item-deleted",
"success",
"subdoc"
],
"desc": "Subdoc: Success, but the affected document was (and still is) deleted",
"name": "SUBDOC_SUCCESS_DELETED"
},
"ce": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: The flag combination doesn't make any sense",
"name": "SUBDOC_XATTR_INVALID_FLAG_COMBO"
},
"cf": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: The key combination of the xattrs is not allowed",
"name": "SUBDOC_XATTR_INVALID_KEY_COMBO"
},
"d0": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: The server don't know about the specified macro",
"name": "SUBDOC_XATTR_UNKNOWN_MACRO"
},
"d1": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: The server don't know about the specified virtual attribute",
"name": "SUBDOC_XATTR_UNKNOWN_VATTR"
},
"d2": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: Can't modify virtual attributes",
"name": "SUBDOC_XATTR_CANT_MODIFY_VATTR"
},
"d3": {
"attrs": [
"item-deleted",
"subdoc",
"special-handling"
],
"desc": "Subdoc: One or more paths in a multi-path command failed on a deleted document",
"name": "SUBDOC_MULTI_PATH_FAILURE_DELETED"
},
"d4": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: Invalid XATTR order (xattrs should come first)",
"name": "SUBDOC_INVALID_XATTR_ORDER"
},
"d5": {
"attrs": [
"subdoc",
"invalid-input"
],
"desc": "Subdoc: The server don't know about (or support) the specified virtual macro",
"name": "SUBDOC_XATTR_UNKNOWN_VATTR_MACRO"
},
"d6": {
"attrs": [
"subdoc",
"item-only"
],
"desc": "Subdoc: Only deleted documents can be revived",
"name": "SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS"
},
"d7": {
"attrs": [
"subdoc",
"item-only"
],
"desc": "Subdoc: A deleted document can't have a value",
"name": "SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE"
}
},
"revision": 1,
"version": 2
}
gocbcore-10.2.3/testdata/query_failure_cas_mismatch_71.json 0000664 0000000 0000000 00000001206 14417540156 0023770 0 ustar 00root root 0000000 0000000 {
"requestID": "9605e383-3da3-440e-a4e1-47d4b673401f",
"clientContextID": "d4c97655-2e89-41ed-a46c-9f4e2a1eae5a",
"signature": {
"*": "*"
},
"results": [],
"errors": [
{
"reason": {
"caller": "couchbase:1978",
"code": 12033,
"key": "datastore.couchbase.CAS_mismatch",
"message": "CAS mismatch"
},
"code": 12009,
"msg": "some other message not matching on cas...",
"retry": false
}
],
"status": "errors",
"metrics": {
"elapsedTime": "1.167435ms",
"executionTime": "1.117429ms",
"resultCount": 0,
"resultSize": 0,
"errorCount": 1
}
}
gocbcore-10.2.3/testdata/query_failure_doc_exists_71.json 0000664 0000000 0000000 00000001034 14417540156 0023500 0 ustar 00root root 0000000 0000000 {
"requestID": "9605e383-3da3-440e-a4e1-47d4b673401f",
"clientContextID": "d4c97655-2e89-41ed-a46c-9f4e2a1eae5a",
"signature": {
"*": "*"
},
"results": [],
"errors": [
{
"reason": {
"code": 17012,
"message": "something kv something"
},
"code": 12009,
"msg": "some message",
"retry": false
}
],
"status": "errors",
"metrics": {
"elapsedTime": "1.167435ms",
"executionTime": "1.117429ms",
"resultCount": 0,
"resultSize": 0,
"errorCount": 1
}
}
gocbcore-10.2.3/testdata/query_failure_doc_not_found_71.json 0000664 0000000 0000000 00000000757 14417540156 0024167 0 ustar 00root root 0000000 0000000 {
"requestID": "9605e383-3da3-440e-a4e1-47d4b673401f",
"clientContextID": "d4c97655-2e89-41ed-a46c-9f4e2a1eae5a",
"signature": {
"*": "*"
},
"results": [],
"errors": [
{
"reason": {
"code": 17014
},
"code": 12009,
"msg": "some message",
"retry": false
}
],
"status": "errors",
"metrics": {
"elapsedTime": "1.167435ms",
"executionTime": "1.117429ms",
"resultCount": 0,
"resultSize": 0,
"errorCount": 1
}
}
gocbcore-10.2.3/testdata/query_failure_retry_true.json 0000664 0000000 0000000 00000000757 14417540156 0023244 0 ustar 00root root 0000000 0000000 {
"requestID": "9605e383-3da3-440e-a4e1-47d4b673401f",
"clientContextID": "d4c97655-2e89-41ed-a46c-9f4e2a1eae5a",
"signature": {
"*": "*"
},
"results": [],
"errors": [
{
"reason": {
"code": 11111
},
"code": 99999,
"msg": "some nonsense",
"retry": true
}
],
"status": "errors",
"metrics": {
"elapsedTime": "1.167435ms",
"executionTime": "1.117429ms",
"resultCount": 0,
"resultSize": 0,
"errorCount": 1
}
}
gocbcore-10.2.3/testdata/query_rows_errors.json 0000664 0000000 0000000 00000001031 14417540156 0021701 0 ustar 00root root 0000000 0000000 {
"requestID": "9605e383-3da3-440e-a4e1-47d4b673401f",
"clientContextID": "d4c97655-2e89-41ed-a46c-9f4e2a1eae5a",
"signature": {
"*": "*"
},
"results": [],
"errors": [
{
"code": 12009,
"msg": "DML Error, possible causes include CAS mismatch or concurrent modificationFailed to perform insert - cause Duplicate Key test"
}
],
"status": "errors",
"metrics": {
"elapsedTime": "1.167435ms",
"executionTime": "1.117429ms",
"resultCount": 0,
"resultSize": 0,
"errorCount": 1
}
}
gocbcore-10.2.3/testdata/query_rows_unknown_errors.json 0000664 0000000 0000000 00000001043 14417540156 0023463 0 ustar 00root root 0000000 0000000 {
"requestID": "9605e383-3da3-440e-a4e1-47d4b673401f",
"clientContextID": "d4c97655-2e89-41ed-a46c-9f4e2a1eae5a",
"signature": {
"*": "*"
},
"results": [],
"errors": [
{
"code": 13014,
"msg": "User does not have credentials to run SELECT queries on the default bucket. Add role query_select on default to allow the query to run."
}
],
"status": "errors",
"metrics": {
"elapsedTime": "1.167435ms",
"executionTime": "1.117429ms",
"resultCount": 0,
"resultSize": 0,
"errorCount": 1
}
}
gocbcore-10.2.3/testdata/search_hits_nil.json 0000664 0000000 0000000 00000001601 14417540156 0021227 0 ustar 00root root 0000000 0000000 {
"status": {
"total": 6,
"failed": 6,
"successful": 0,
"errors": {
"travel_a464a32f957f35f1_13aa53f3": "context deadline exceeded",
"travel_a464a32f957f35f1_18572d87": "context deadline exceeded",
"travel_a464a32f957f35f1_54820232": "context deadline exceeded",
"travel_a464a32f957f35f1_6ddbfb54": "context deadline exceeded",
"travel_a464a32f957f35f1_aa574717": "context deadline exceeded",
"travel_a464a32f957f35f1_f4e0a48a": "context deadline exceeded"
}
},
"request": {
"query": {
"boost": null,
"match_all": {}
},
"size": 1000,
"from": 0,
"highlight": null,
"fields": [
"*"
],
"facets": null,
"explain": false,
"sort": [
"-_score"
],
"includeLocations": false
},
"hits": null,
"total_hits": 0,
"max_score": 0,
"took": 32464205,
"facets": null
}
gocbcore-10.2.3/testdcpobserver_test.go 0000664 0000000 0000000 00000005625 14417540156 0020203 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"strconv"
"sync"
)
type DCPEventCounter struct {
mutations map[string]DcpMutation
deletions map[string]DcpDeletion
expirations map[string]DcpExpiration
scopes map[string]int
collections map[string]int
scopesDeleted map[string]int
collectionsDeleted map[string]int
}
type TestStreamObserver struct {
lock sync.Mutex
lastSeqno map[uint16]uint64
snapshots map[uint16]DcpSnapshotMarker
counter *DCPEventCounter
endWg sync.WaitGroup
}
func (so *TestStreamObserver) newCounter() {
so.counter = &DCPEventCounter{
mutations: make(map[string]DcpMutation),
deletions: make(map[string]DcpDeletion),
expirations: make(map[string]DcpExpiration),
scopes: make(map[string]int),
collections: make(map[string]int),
scopesDeleted: make(map[string]int),
collectionsDeleted: make(map[string]int),
}
}
func (so *TestStreamObserver) SnapshotMarker(snapshotMarker DcpSnapshotMarker) {
so.lock.Lock()
so.snapshots[snapshotMarker.VbID] = snapshotMarker
if so.lastSeqno[snapshotMarker.VbID] < snapshotMarker.StartSeqNo || so.lastSeqno[snapshotMarker.VbID] > snapshotMarker.EndSeqNo {
so.lastSeqno[snapshotMarker.VbID] = snapshotMarker.StartSeqNo
}
so.lock.Unlock()
}
func (so *TestStreamObserver) Mutation(mutation DcpMutation) {
so.lock.Lock()
so.counter.mutations[string(mutation.Key)] = mutation
so.lock.Unlock()
}
func (so *TestStreamObserver) Deletion(deletion DcpDeletion) {
so.lock.Lock()
so.counter.deletions[string(deletion.Key)] = deletion
so.lock.Unlock()
}
func (so *TestStreamObserver) Expiration(expiration DcpExpiration) {
so.lock.Lock()
so.counter.expirations[string(expiration.Key)] = expiration
so.lock.Unlock()
}
func (so *TestStreamObserver) End(end DcpStreamEnd, err error) {
so.endWg.Done()
}
func (so *TestStreamObserver) CreateCollection(creation DcpCollectionCreation) {
so.lock.Lock()
so.counter.collections[strconv.Itoa(int(creation.ScopeID))+"."+string(creation.Key)]++
so.lock.Unlock()
}
func (so *TestStreamObserver) DeleteCollection(deletion DcpCollectionDeletion) {
so.lock.Lock()
so.counter.collectionsDeleted[strconv.Itoa(int(deletion.ScopeID))+"."+strconv.Itoa(int(deletion.CollectionID))]++
so.lock.Unlock()
}
func (so *TestStreamObserver) FlushCollection(flush DcpCollectionFlush) {
}
func (so *TestStreamObserver) CreateScope(creation DcpScopeCreation) {
so.lock.Lock()
so.counter.scopes[string(creation.Key)]++
so.lock.Unlock()
}
func (so *TestStreamObserver) DeleteScope(deletion DcpScopeDeletion) {
so.lock.Lock()
so.counter.scopesDeleted[strconv.Itoa(int(deletion.ScopeID))]++
so.lock.Unlock()
}
func (so *TestStreamObserver) ModifyCollection(modification DcpCollectionModification) {
}
func (so *TestStreamObserver) OSOSnapshot(snapshot DcpOSOSnapshot) {
}
func (so *TestStreamObserver) SeqNoAdvanced(seqNoAdvanced DcpSeqNoAdvanced) {
}
gocbcore-10.2.3/testdcpsuite_test.go 0000664 0000000 0000000 00000056201 14417540156 0017501 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"context"
"crypto/x509"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/couchbase/gocbcore/v10/memd"
"github.com/stretchr/testify/suite"
)
type DCPTestSuite struct {
suite.Suite
*DCPTestConfig
opAgent *Agent
dcpAgent *DCPAgent
so *TestStreamObserver
}
func (suite *DCPTestSuite) SupportsFeature(feature TestFeatureCode) bool {
featureFlagValue := 0
for _, featureFlag := range suite.FeatureFlags {
if featureFlag.Feature == feature || featureFlag.Feature == "*" {
if featureFlag.Enabled {
featureFlagValue = +1
} else {
featureFlagValue = -1
}
}
}
if featureFlagValue == -1 {
return false
} else if featureFlagValue == +1 {
return true
}
switch feature {
case TestFeatureDCPExpiry:
return !suite.ClusterVersion.Lower(srvVer650)
case TestFeatureDCPDeleteTimes:
return !suite.ClusterVersion.Lower(srvVer650)
case TestFeatureCollections:
return !suite.ClusterVersion.Lower(srvVer700)
}
panic("found unsupported feature code")
}
func (suite *DCPTestSuite) EnsureSupportsFeature(feature TestFeatureCode) {
if !suite.SupportsFeature(feature) {
suite.T().Skipf("Skipping test due to disabled feature code: %s", feature)
}
}
func (suite *DCPTestSuite) SetupSuite() {
suite.DCPTestConfig = globalDCPTestConfig
suite.Require().NotEmpty(suite.DCPTestConfig.ConnStr, "Connection string cannot be empty for testing DCP")
var err error
suite.opAgent, err = suite.initAgent(suite.makeOpAgentConfig(suite.DCPTestConfig))
suite.Require().Nil(err)
flags := memd.DcpOpenFlagProducer
if suite.SupportsFeature(TestFeatureDCPDeleteTimes) {
flags |= memd.DcpOpenFlagIncludeDeleteTimes
}
suite.dcpAgent, err = suite.initDCPAgent(
suite.makeDCPAgentConfig(suite.DCPTestConfig, suite.SupportsFeature(TestFeatureDCPExpiry)),
"test-stream",
flags,
)
suite.Require().Nil(err, err)
suite.so = &TestStreamObserver{
lock: sync.Mutex{},
lastSeqno: make(map[uint16]uint64),
snapshots: make(map[uint16]DcpSnapshotMarker),
endWg: sync.WaitGroup{},
}
}
func (suite *DCPTestSuite) TearDownSuite() {
if suite.opAgent != nil {
suite.opAgent.Close()
suite.opAgent = nil
}
if suite.dcpAgent != nil {
suite.dcpAgent.Close()
suite.dcpAgent = nil
}
}
func (suite *DCPTestSuite) makeOpAgentConfig(testConfig *DCPTestConfig) AgentConfig {
config := AgentConfig{}
config.FromConnStr(testConfig.ConnStr)
config.IoConfig.UseMutationTokens = true
config.IoConfig.UseCollections = true
config.BucketName = testConfig.BucketName
config.SecurityConfig.Auth = testConfig.Authenticator
if config.SecurityConfig.UseTLS {
if testConfig.CAProvider == nil {
config.SecurityConfig.TLSRootCAProvider = func() *x509.CertPool {
return nil
}
} else {
config.SecurityConfig.TLSRootCAProvider = testConfig.CAProvider
}
}
return config
}
func (suite *DCPTestSuite) initAgent(config AgentConfig) (*Agent, error) {
agent, err := CreateAgent(&config)
if err != nil {
return nil, err
}
ch := make(chan error)
_, err = agent.WaitUntilReady(
time.Now().Add(10*time.Second),
WaitUntilReadyOptions{},
func(result *WaitUntilReadyResult, err error) {
ch <- err
},
)
if err != nil {
return nil, err
}
err = <-ch
if err != nil {
return nil, err
}
return agent, nil
}
func (suite *DCPTestSuite) makeDCPAgentConfig(testConfig *DCPTestConfig, expiryEnabled bool) DCPAgentConfig {
config := DCPAgentConfig{}
config.FromConnStr(testConfig.ConnStr)
config.IoConfig.UseCollections = suite.SupportsFeature(TestFeatureCollections)
config.BucketName = testConfig.BucketName
if expiryEnabled {
config.DCPConfig.UseExpiryOpcode = true
}
config.SecurityConfig.Auth = testConfig.Authenticator
if config.SecurityConfig.UseTLS {
if testConfig.CAProvider == nil {
config.SecurityConfig.TLSRootCAProvider = func() *x509.CertPool {
return nil
}
} else {
config.SecurityConfig.TLSRootCAProvider = testConfig.CAProvider
}
}
return config
}
func (suite *DCPTestSuite) initDCPAgent(config DCPAgentConfig, streamName string, openFlags memd.DcpOpenFlag) (*DCPAgent, error) {
agent, err := CreateDcpAgent(&config, streamName, openFlags)
if err != nil {
return nil, err
}
ch := make(chan error)
_, err = agent.WaitUntilReady(
time.Now().Add(10*time.Second),
WaitUntilReadyOptions{},
func(result *WaitUntilReadyResult, err error) {
ch <- err
},
)
if err != nil {
return nil, err
}
err = <-ch
if err != nil {
return nil, err
}
return agent, nil
}
func TestDCPSuite(t *testing.T) {
if globalDCPTestConfig == nil {
t.Skip()
}
suite.Run(t, new(DCPTestSuite))
}
func (suite *DCPTestSuite) runMutations(collection, scope string) (map[string]string, []string) {
expirationsLeft := suite.NumExpirations
deletionsLeft := int32(suite.NumDeletions)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var wg sync.WaitGroup
wg.Add(suite.NumMutations)
mutations := make(map[string]string)
var deletionKeys []string
lock := new(sync.Mutex)
for i := 0; i < suite.NumMutations; i++ {
var expiry uint32
if expirationsLeft > 0 {
expiry = 1
expirationsLeft--
}
var doMutation func(ex uint32, id int)
doMutation = func(ex uint32, id int) {
var success bool
defer func() {
if success {
wg.Done()
}
}()
ch := make(chan error, 1)
op, err := suite.opAgent.Set(
SetOptions{
Key: []byte(fmt.Sprintf("key-%d", id)),
Value: []byte(fmt.Sprintf("value-%d", id)),
Expiry: ex,
ScopeName: scope,
CollectionName: collection,
Deadline: time.Now().Add(1000 * time.Millisecond),
}, func(result *StoreResult, err error) {
ch <- err
},
)
if err != nil {
suite.T().Logf("Retrying due to failure sending set: %v", err)
doMutation(ex, id)
return
}
select {
case err := <-ch:
if err != nil {
suite.T().Logf("Retrying due to failure performing set: %v", err)
doMutation(ex, id)
return
}
case <-ctx.Done():
op.Cancel()
return
}
if ex == 0 {
dLeft := atomic.AddInt32(&deletionsLeft, -1)
if dLeft >= 0 {
lock.Lock()
deletionKeys = append(deletionKeys, fmt.Sprintf("key-%d", id))
lock.Unlock()
ch = make(chan error, 1)
op, err := suite.opAgent.Delete(
DeleteOptions{
Key: []byte(fmt.Sprintf("key-%d", id)),
CollectionName: collection,
ScopeName: scope,
Deadline: time.Now().Add(2500 * time.Millisecond),
}, func(result *DeleteResult, err error) {
ch <- err
},
)
if err != nil {
suite.T().Logf("Retrying due to failure sending delete: %v", err)
doMutation(ex, id)
return
}
select {
case err := <-ch:
if err != nil {
suite.T().Logf("Retrying due to failure performing delete: %v", err)
doMutation(ex, id)
return
}
case <-ctx.Done():
op.Cancel()
return
}
} else {
lock.Lock()
mutations[fmt.Sprintf("key-%d", id)] = fmt.Sprintf("value-%d", id)
lock.Unlock()
}
}
success = true
}
go doMutation(expiry, i)
}
wgCh := make(chan struct{}, 1)
go func() {
wg.Wait()
wgCh <- struct{}{}
}()
select {
case <-ctx.Done():
suite.T().Fatalf("Failed to perform mutations due to %v", ctx.Err())
case <-wgCh:
cancel()
// Let any expirations do their thing
time.Sleep(5 * time.Second)
}
return mutations, deletionKeys
}
func (suite *DCPTestSuite) getFailoverLogs(nVB int, dcpAgent *DCPAgent) (map[int]FailoverEntry, error) {
ch := make(chan error)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
failOverEntries := make(map[int]FailoverEntry)
var openWg sync.WaitGroup
openWg.Add(nVB)
lock := sync.Mutex{}
for i := 0; i < nVB; i++ {
go func(vbId uint16) {
op, err := dcpAgent.GetFailoverLog(vbId, func(entries []FailoverEntry, err error) {
for _, en := range entries {
lock.Lock()
failOverEntries[int(vbId)] = en
lock.Unlock()
}
ch <- err
})
if err != nil {
cancel()
return
}
select {
case err := <-ch:
if err != nil {
fmt.Printf("Error received from get failover logs: %v", err)
cancel()
return
}
case <-ctx.Done():
op.Cancel()
return
}
openWg.Done()
}(uint16(i))
}
wgCh := make(chan struct{}, 1)
go func() {
openWg.Wait()
wgCh <- struct{}{}
}()
select {
case <-ctx.Done():
return nil, errors.New("Failed to get failoverlogs")
case <-wgCh:
cancel()
}
return failOverEntries, nil
}
//Runs a dcp stream on all VBs from the last snapshot to the current seqno
func (suite *DCPTestSuite) runDCPStream(dcpAgent *DCPAgent) int {
suite.so.newCounter()
seqnos, err := suite.getCurrentSeqNos(dcpAgent)
suite.Require().Nil(err, err)
suite.so.endWg.Add(len(seqnos))
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var openWg sync.WaitGroup
openWg.Add(len(seqnos))
fo, err := suite.getFailoverLogs(len(seqnos), dcpAgent)
suite.Require().Nil(err, err)
//Start streaming from all VBs from the latest snapshot, until the current seqno
for _, entry := range seqnos {
go func(en VbSeqNoEntry) {
ch := make(chan error, 1)
suite.so.lock.Lock()
snapshot := suite.so.snapshots[en.VbID]
suite.so.lock.Unlock()
op, err := dcpAgent.OpenStream(en.VbID, memd.DcpStreamAddFlagActiveOnly, fo[int(en.VbID)].VbUUID, SeqNo(snapshot.EndSeqNo), en.SeqNo,
SeqNo(snapshot.StartSeqNo), SeqNo(snapshot.EndSeqNo), suite.so, OpenStreamOptions{}, func(entries []FailoverEntry, err error) {
ch <- err
},
)
if err != nil {
cancel()
return
}
select {
case err := <-ch:
if err != nil {
suite.T().Logf("Error received from open stream: %v", err)
cancel()
return
}
case <-ctx.Done():
op.Cancel()
return
}
openWg.Done()
}(entry)
}
wgCh := make(chan struct{}, 1)
go func() {
openWg.Wait()
wgCh <- struct{}{}
}()
select {
case <-ctx.Done():
suite.T().Fatal("Failed to open streams")
case <-wgCh:
cancel()
// Let any expirations do their thing
time.Sleep(5 * time.Second)
}
suite.T().Logf("All streams open, waiting for streams to complete")
waitCh := make(chan struct{})
go func() {
suite.so.endWg.Wait()
close(waitCh)
}()
select {
case <-time.After(60 * time.Second):
suite.T().Fatal("Timed out waiting for streams to complete")
case <-waitCh:
}
suite.T().Logf("All streams complete")
return len(seqnos)
}
func (suite *DCPTestSuite) getCurrentSeqNos(dcpAgent *DCPAgent) ([]VbSeqNoEntry, error) {
h := makeTestSubHarness(suite.T())
var seqnos []VbSeqNoEntry
snapshot, err := dcpAgent.ConfigSnapshot()
suite.Require().Nil(err, err)
numNodes, err := snapshot.NumServers()
suite.Require().Nil(err, err)
//Get all SeqNos
for i := 1; i < numNodes+1; i++ {
h.PushOp(dcpAgent.GetVbucketSeqnos(i, memd.VbucketStateActive, GetVbucketSeqnoOptions{},
func(entries []VbSeqNoEntry, err error) {
h.Wrap(func() {
if err != nil {
h.Fatalf("GetVbucketSeqnos operation failed: %v", err)
return
}
seqnos = append(seqnos, entries...)
})
}))
h.Wait(0)
}
return seqnos, nil
}
func (suite *DCPTestSuite) TestBasic() {
mutations, deletionKeys := suite.runMutations("", "")
suite.runDCPStream(suite.dcpAgent)
// Compaction can run and cause expirations to be hidden from us
suite.Assert().InDelta(suite.NumMutations, len(suite.so.counter.mutations), float64(suite.NumExpirations))
suite.Assert().Equal(suite.NumDeletions, len(suite.so.counter.deletions))
suite.Assert().InDelta(suite.NumExpirations, len(suite.so.counter.expirations), float64(suite.NumExpirations))
for key, val := range mutations {
if suite.Assert().Contains(suite.so.counter.mutations, key) {
suite.Assert().Equal(string(suite.so.counter.mutations[key].Value), val)
}
}
for _, key := range deletionKeys {
suite.Assert().Contains(suite.so.counter.deletions, key)
}
}
func (suite *DCPTestSuite) TestScopesBasic() {
suite.EnsureSupportsFeature(TestFeatureCollections)
prefix := "dcp_scope_sbasic"
scopes := suite.makeScopes(suite.NumScopes, prefix, suite.BucketName, suite.opAgent)
nVB := suite.runDCPStream(suite.dcpAgent)
pScopes := suite.getPrunedScopeManifests(prefix, scopes)
suite.Assert().Equal(len(pScopes), len(suite.so.counter.scopes))
for _, val := range pScopes {
suite.Assert().Equal(nVB, suite.so.counter.scopes[val.Name])
}
}
func (suite *DCPTestSuite) TestScopesDrops() {
suite.EnsureSupportsFeature(TestFeatureCollections)
//Make scopes
prefix := "dcp_scope_sdrops"
scopes := suite.makeScopes(suite.NumScopes, prefix, suite.BucketName, suite.opAgent)
//Drop all scopes created in this test
pScopes := suite.getPrunedScopeManifests(prefix, scopes)
suite.dropScopes(pScopes, suite.BucketName, suite.opAgent)
nVB := suite.runDCPStream(suite.dcpAgent)
if !suite.Assert().Equal(suite.NumScopes, len(suite.so.counter.scopesDeleted)) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopesDeleted, suite.so.counter.collectionsDeleted)
dumpManifest(suite.opAgent, suite.T())
}
for _, val := range pScopes {
if !suite.Assert().Equal(nVB, suite.so.counter.scopesDeleted[strconv.Itoa(int(val.UID))], fmt.Sprintf("For scope %s", val.Name)) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopesDeleted, suite.so.counter.collectionsDeleted)
dumpManifest(suite.opAgent, suite.T())
}
}
}
func (suite *DCPTestSuite) TestCollectionsBasic() {
suite.EnsureSupportsFeature(TestFeatureCollections)
//Make scopes
prefix := "dcp_scope_cbasic"
scopes := suite.makeScopes(suite.NumScopes, prefix, suite.BucketName, suite.opAgent)
//Make NumCollections per scope
pScopes := suite.getPrunedScopeManifests(prefix, scopes)
lastScopeManifest := suite.makeCollections(suite.NumCollections, "dcp_collection_cbasic", pScopes, suite.BucketName, suite.opAgent)
pScopes = suite.getPrunedScopeManifests(prefix, lastScopeManifest)
nVB := suite.runDCPStream(suite.dcpAgent)
if !suite.Assert().Equal(suite.NumScopes, len(suite.so.counter.scopes)) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopes, suite.so.counter.collections)
dumpManifest(suite.opAgent, suite.T())
}
if !suite.Assert().Equal(suite.NumCollections*suite.NumScopes, len(suite.so.counter.collections)) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopes, suite.so.counter.collections)
dumpManifest(suite.opAgent, suite.T())
}
for _, val := range pScopes {
suite.Assert().Equal(nVB, suite.so.counter.scopes[val.Name])
for _, c := range val.Collections {
if !suite.Assert().Equal(nVB, suite.so.counter.collections[strconv.Itoa(int(val.UID))+"."+c.Name]) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopes, suite.so.counter.collections)
dumpManifest(suite.opAgent, suite.T())
}
}
}
}
func (suite *DCPTestSuite) TestCollectionsScopeDrop() {
suite.EnsureSupportsFeature(TestFeatureCollections)
//Make scopes
prefix := "dcp_scope_csdrop"
scopes := suite.makeScopes(suite.NumScopes, prefix, suite.BucketName, suite.opAgent)
//Make NumCollections per scope
pScopes := suite.getPrunedScopeManifests(prefix, scopes)
lastScopeManifest := suite.makeCollections(suite.NumCollections, "dcp_collection_csdrop", pScopes, suite.BucketName, suite.opAgent)
pScopes = suite.getPrunedScopeManifests(prefix, lastScopeManifest)
//Drop all scopes created in this test, implicitly dropping all the collections
suite.dropScopes(pScopes, suite.BucketName, suite.opAgent)
nVB := suite.runDCPStream(suite.dcpAgent)
if !suite.Assert().Equal(suite.NumScopes, len(suite.so.counter.scopesDeleted)) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopesDeleted, suite.so.counter.collectionsDeleted)
dumpManifest(suite.opAgent, suite.T())
}
if !suite.Assert().Equal(suite.NumCollections*suite.NumScopes, len(suite.so.counter.collectionsDeleted)) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopesDeleted, suite.so.counter.collectionsDeleted)
dumpManifest(suite.opAgent, suite.T())
}
for _, s := range pScopes {
suite.Assert().Equal(nVB, suite.so.counter.scopesDeleted[strconv.Itoa(int(s.UID))], fmt.Sprintf("For scope %s", s.Name))
for _, c := range s.Collections {
if !suite.Assert().Equal(nVB, suite.so.counter.collectionsDeleted[strconv.Itoa(int(s.UID))+"."+strconv.Itoa(int(c.UID))]) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopesDeleted, suite.so.counter.collectionsDeleted)
dumpManifest(suite.opAgent, suite.T())
}
}
}
}
func (suite *DCPTestSuite) TestCollectionsDrop() {
suite.EnsureSupportsFeature(TestFeatureCollections)
//Make scopes
prefix := "dcp_scope_ccdrop"
scopes := suite.makeScopes(suite.NumScopes, prefix, suite.BucketName, suite.opAgent)
//Make NumCollections per scope
pScopes := suite.getPrunedScopeManifests(prefix, scopes)
lastScopeManifest := suite.makeCollections(suite.NumCollections, "dcp_collection_ccdrop", pScopes, suite.BucketName, suite.opAgent)
pScopes = suite.getPrunedScopeManifests(prefix, lastScopeManifest)
//Drop all collections created in this test
suite.dropCollections(pScopes, suite.BucketName, suite.opAgent)
nVB := suite.runDCPStream(suite.dcpAgent)
if !suite.Assert().Equal(suite.NumScopes, len(suite.so.counter.scopes)) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopesDeleted, suite.so.counter.collectionsDeleted)
dumpManifest(suite.opAgent, suite.T())
}
if !suite.Assert().Equal(suite.NumCollections*suite.NumScopes, len(suite.so.counter.collectionsDeleted)) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopesDeleted, suite.so.counter.collectionsDeleted)
dumpManifest(suite.opAgent, suite.T())
}
for _, s := range pScopes {
suite.Assert().Equal(nVB, suite.so.counter.scopes[s.Name])
for _, c := range s.Collections {
if !suite.Assert().Equal(nVB, suite.so.counter.collectionsDeleted[strconv.Itoa(int(s.UID))+"."+strconv.Itoa(int(c.UID))]) {
suite.T().Logf("Scopes: %+v, Collections: %+v", suite.so.counter.scopesDeleted, suite.so.counter.collectionsDeleted)
dumpManifest(suite.opAgent, suite.T())
}
}
}
}
func (suite *DCPTestSuite) TestMutationsCollection() {
suite.EnsureSupportsFeature(TestFeatureCollections)
// Make scopes
sPrefix := "dcp_scope_mut"
cPrefix := "dcp_collection_mut"
scopes := suite.makeScopes(suite.NumScopes, sPrefix, suite.BucketName, suite.opAgent)
// Make NumCollections per scope
pScopes := suite.getPrunedScopeManifests(sPrefix, scopes)
lastScopeManifest := suite.makeCollections(suite.NumCollections, cPrefix, pScopes, suite.BucketName, suite.opAgent)
suite.getPrunedScopeManifests(sPrefix, lastScopeManifest)
time.Sleep(5 * time.Second) // Needed to ensure collection ready before performing mutations.
mutations, deletionKeys := suite.runMutations(cPrefix+"0", sPrefix+"0")
suite.runDCPStream(suite.dcpAgent)
// Compaction can run and cause expirations to be hidden from us
suite.Assert().InDelta(suite.NumMutations, len(suite.so.counter.mutations), float64(suite.NumExpirations))
suite.Assert().Equal(suite.NumDeletions, len(suite.so.counter.deletions))
suite.Assert().InDelta(suite.NumExpirations, len(suite.so.counter.expirations), float64(suite.NumExpirations))
for key, val := range mutations {
if suite.Assert().Contains(suite.so.counter.mutations, key) {
suite.Assert().Equal(string(suite.so.counter.mutations[key].Value), val)
}
}
for _, key := range deletionKeys {
suite.Assert().Contains(suite.so.counter.deletions, key)
}
}
func (suite *DCPTestSuite) TestNSAgent() {
flags := memd.DcpOpenFlagProducer
if suite.SupportsFeature(TestFeatureDCPDeleteTimes) {
flags |= memd.DcpOpenFlagIncludeDeleteTimes
}
srcCfg := suite.makeDCPAgentConfig(suite.DCPTestConfig, suite.SupportsFeature(TestFeatureDCPExpiry))
if len(srcCfg.SeedConfig.HTTPAddrs) == 0 {
suite.T().Skip("Skipping test due to no HTTP addresses")
}
seedAddr := srcCfg.SeedConfig.HTTPAddrs[0]
parts := strings.Split(seedAddr, ":")
if parts[1] != "8091" && parts[1] != "11210" {
// This should work with non default ports but it makes the test logic too complicated.
// This implicitly means that if TLS is enabled then this test won't run.
suite.T().Skip("Skipping test due to non default ports have been supplied")
}
connstr := fmt.Sprintf("ns_server://%s", seedAddr)
config := DCPAgentConfig{
BucketName: srcCfg.BucketName,
}
err := config.FromConnStr(connstr)
suite.Require().Nil(err, err)
config.IoConfig = srcCfg.IoConfig
config.DCPConfig = srcCfg.DCPConfig
config.SecurityConfig.TLSRootCAProvider = func() *x509.CertPool {
return nil
}
config.SecurityConfig.Auth = srcCfg.SecurityConfig.Auth
config.SecurityConfig.AuthMechanisms = srcCfg.SecurityConfig.AuthMechanisms
dcpAgent, err := suite.initDCPAgent(config, "ns-stream", flags)
suite.Require().Nil(err, err)
defer dcpAgent.Close()
mutations, deletionKeys := suite.runMutations("", "")
suite.runDCPStream(dcpAgent)
// Compaction can run and cause expirations to be hidden from us
suite.Assert().InDelta(suite.NumMutations, len(suite.so.counter.mutations), float64(suite.NumExpirations))
suite.Assert().Equal(suite.NumDeletions, len(suite.so.counter.deletions))
suite.Assert().InDelta(suite.NumExpirations, len(suite.so.counter.expirations), float64(suite.NumExpirations))
for key, val := range mutations {
if suite.Assert().Contains(suite.so.counter.mutations, key) {
suite.Assert().Equal(string(suite.so.counter.mutations[key].Value), val)
}
}
for _, key := range deletionKeys {
suite.Assert().Contains(suite.so.counter.deletions, key)
}
}
func (suite *DCPTestSuite) makeScopes(n int, prefix, bucketName string, agent *Agent) []ManifestScope {
var scopes []string
var err error
var m *Manifest
for i := 0; i < n; i++ {
s := prefix + strconv.Itoa(i)
scopes = append(scopes, s)
m, err = testCreateScope(s, bucketName, agent)
suite.Require().Nil(err, err)
}
return m.Scopes
}
//Return only the scope manifests with the provided prefix name
func (suite *DCPTestSuite) getPrunedScopeManifests(prefix string, sm []ManifestScope) []ManifestScope {
var prunedScopes []ManifestScope
for _, s := range sm {
match, err := regexp.Match(prefix+"+", []byte(s.Name))
suite.Require().Nil(err, err)
if s.Name != "_default" && match {
prunedScopes = append(prunedScopes, s)
}
}
return prunedScopes
}
func (suite *DCPTestSuite) dropScopes(scopes []ManifestScope, bucketName string, agent *Agent) {
for _, s := range scopes {
_, err := testDeleteScope(s.Name, bucketName, agent, true)
suite.Require().Nil(err, err)
}
}
func (suite *DCPTestSuite) dropCollections(scopes []ManifestScope, bucketName string, agent *Agent) {
for _, s := range scopes {
for _, c := range s.Collections {
_, err := testDeleteCollection(c.Name, s.Name, bucketName, agent, true)
suite.Require().Nil(err, err)
}
}
}
func (suite *DCPTestSuite) makeCollections(n int, prefix string, scopes []ManifestScope, bucketName string, agent *Agent) []ManifestScope {
var m *Manifest
var err error
for _, s := range scopes {
for i := 0; i < n; i++ {
c := prefix + strconv.Itoa(i)
m, err = testCreateCollection(c, s.Name, bucketName, agent)
suite.Require().Nil(err, err)
}
}
return m.Scopes
}
gocbcore-10.2.3/testdocs_test.go 0000664 0000000 0000000 00000003130 14417540156 0016602 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"fmt"
"testing"
)
const TestNumDocs = 5
type testDoc struct {
TestName string `json:"testName"`
I int `json:"i"`
}
type testDocs struct {
t *testing.T
agent *Agent
testName string
numDocs int
}
func (td *testDocs) upsert() {
waitCh := make(chan error, td.numDocs)
for i := 1; i <= td.numDocs; i++ {
testDocName := fmt.Sprintf("%s-%d", td.testName, i)
bytes, err := json.Marshal(testDoc{
TestName: td.testName,
I: i,
})
if err != nil {
td.t.Errorf("failed to marshal test doc: %v", err)
return
}
_, err = td.agent.Set(SetOptions{
Key: []byte(testDocName),
Value: bytes,
}, func(res *StoreResult, err error) {
waitCh <- err
})
if err != nil {
td.t.Errorf("failed to set test doc: %v", err)
}
}
for i := 1; i <= td.numDocs; i++ {
err := <-waitCh
if err != nil {
td.t.Errorf("failed to remove test doc: %v", err)
return
}
}
}
func (td *testDocs) Remove() {
waitCh := make(chan error, td.numDocs)
for i := 1; i <= td.numDocs; i++ {
testDocName := fmt.Sprintf("%s-%d", td.testName, i)
td.agent.Delete(DeleteOptions{
Key: []byte(testDocName),
}, func(res *DeleteResult, err error) {
waitCh <- err
})
}
for i := 1; i <= td.numDocs; i++ {
err := <-waitCh
if err != nil {
td.t.Errorf("failed to remove test doc: %v", err)
return
}
}
}
func makeTestDocs(t *testing.T, agent *Agent, testName string, numDocs int) *testDocs {
td := &testDocs{
t: t,
agent: agent,
testName: testName,
numDocs: numDocs,
}
td.upsert()
return td
}
gocbcore-10.2.3/testharness_test.go 0000664 0000000 0000000 00000015577 14417540156 0017337 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
"time"
)
var (
srvVer450 = NodeVersion{4, 5, 0, 0, 0, ""}
srvVer551 = NodeVersion{5, 5, 1, 0, 0, ""}
srvVer552 = NodeVersion{5, 5, 2, 0, 0, ""}
srvVer553 = NodeVersion{5, 5, 3, 0, 0, ""}
srvVer600 = NodeVersion{6, 0, 0, 0, 0, ""}
srvVer650 = NodeVersion{6, 5, 0, 0, 0, ""}
srvVer650DP = NodeVersion{6, 5, 0, 0, 0, "dp"}
srvVer660 = NodeVersion{6, 6, 0, 0, 0, ""}
srvVer700 = NodeVersion{7, 0, 0, 0, 0, ""}
srvVer710 = NodeVersion{7, 1, 0, 0, 0, ""}
srvVer720 = NodeVersion{7, 2, 0, 0, 0, ""}
srvVer720DP = NodeVersion{7, 2, 0, 0, 0, "dp"}
srvVer750 = NodeVersion{7, 5, 0, 0, 0, ""}
mockVer156 = NodeVersion{1, 5, 6, 0, 0, ""}
)
type TestFeatureCode string
var (
TestFeatureReplicas = TestFeatureCode("replicas")
TestFeatureSsl = TestFeatureCode("ssl")
TestFeatureViews = TestFeatureCode("views")
TestFeatureN1ql = TestFeatureCode("n1ql")
TestFeatureCbas = TestFeatureCode("cbas")
TestFeatureFts = TestFeatureCode("fts")
TestFeatureErrMap = TestFeatureCode("errmap")
TestFeatureCollections = TestFeatureCode("collections")
TestFeatureDCPExpiry = TestFeatureCode("dcpexpiry")
TestFeatureDCPDeleteTimes = TestFeatureCode("dcpdeletetimes")
TestFeatureMemd = TestFeatureCode("memd")
TestFeatureGetMeta = TestFeatureCode("getmeta")
TestFeatureGCCCP = TestFeatureCode("gcccp")
TestFeatureEnhancedDurability = TestFeatureCode("durability")
TestFeatureCreateDeleted = TestFeatureCode("createasdeleted")
TestFeatureReplaceBodyWithXattr = TestFeatureCode("replacebodywithxattr")
TestFeatureExpandMacros = TestFeatureCode("expandmacros")
TestFeaturePreserveExpiry = TestFeatureCode("preserveexpiry")
TestFeatureExpandMacrosSeqNo = TestFeatureCode("expandmacrosseqno")
TestFeatureTransactions = TestFeatureCode("transactions")
TestFeatureN1qlReasons = TestFeatureCode("n1qlreasons")
TestFeatureResourceUnits = TestFeatureCode("computeunits")
TestFeatureRangeScan = TestFeatureCode("rangescan")
)
type TestFeatureFlag struct {
Enabled bool
Feature TestFeatureCode
}
func ParseFeatureFlags(featuresToTest string) []TestFeatureFlag {
var featureFlags []TestFeatureFlag
featureFlagStrs := strings.Split(featuresToTest, ",")
for _, featureFlagStr := range featureFlagStrs {
if len(featureFlagStr) == 0 {
continue
}
if featureFlagStr[0] == '+' {
featureFlags = append(featureFlags, TestFeatureFlag{
Enabled: true,
Feature: TestFeatureCode(featureFlagStr[1:]),
})
continue
} else if featureFlagStr[0] == '-' {
featureFlags = append(featureFlags, TestFeatureFlag{
Enabled: false,
Feature: TestFeatureCode(featureFlagStr[1:]),
})
continue
}
panic("failed to parse specified feature codes")
}
return featureFlags
}
func ParseCerts(path string) (*x509.CertPool, *tls.Certificate, error) {
ca, err := ioutil.ReadFile(path + "/ca.pem")
if err != nil {
return nil, nil, err
}
roots := x509.NewCertPool()
roots.AppendCertsFromPEM(ca)
clientCert, err := ioutil.ReadFile(path + "/client.pem")
if err == os.ErrNotExist {
return roots, nil, nil
} else if err != nil {
return nil, nil, err
}
clientKey, err := ioutil.ReadFile(path + "/client.key")
if err != nil {
return nil, nil, err
}
cert, err := tls.X509KeyPair(clientCert, clientKey)
if err != nil {
return nil, nil, err
}
return roots, &cert, nil
}
// Gets a set of keys evenly distributed across all server nodes.
// the result is an array of strings, each index representing an index
// of a server
func MakeDistKeys(agent *Agent, deadline time.Time) (keys []string, errOut error) {
// Get the routing information
// We can't make dist keys until we're connected.
var clientMux *kvMuxState
for {
clientMux = agent.kvMux.getState()
if clientMux.RevID() > -1 {
break
}
select {
case <-time.After(time.Until(deadline)):
errOut = errTimeout
return
case <-time.After(time.Millisecond):
}
}
keys = make([]string, clientMux.NumPipelines())
remaining := len(keys)
for i := 0; remaining > 0; i++ {
keyTmp := fmt.Sprintf("DistKey_%d", i)
// Map the vBucket and server
vbID := clientMux.VBMap().VbucketByKey([]byte(keyTmp))
srvIx, err := clientMux.VBMap().NodeByVbucket(vbID, 0)
if err != nil || srvIx < 0 || srvIx >= len(keys) || keys[srvIx] != "" {
continue
}
keys[srvIx] = keyTmp
remaining--
}
return
}
type TestSubHarnessBase struct {
sigT testing.TB
sigCh chan int
}
func (h *TestSubHarnessBase) Continue() {
h.sigCh <- 0
}
func (h *TestSubHarnessBase) Wrap(fn func()) {
defer func() {
if r := recover(); r != nil {
// Rethrow actual panics
if r != h {
panic(r)
}
}
}()
fn()
h.sigCh <- 0
}
func (h *TestSubHarnessBase) Fatalf(fmt string, args ...interface{}) {
h.sigT.Helper()
h.sigT.Logf(fmt, args...)
h.sigCh <- 1
panic(h)
}
func (h *TestSubHarnessBase) Skipf(fmt string, args ...interface{}) {
h.sigT.Helper()
h.sigT.Logf(fmt, args...)
h.sigCh <- 2
panic(h)
}
func (h *TestSubHarnessBase) Wait(waitSecs int, cb func(bool)) {
if waitSecs <= 0 {
waitSecs = 5
}
select {
case v := <-h.sigCh:
cb(true)
if v == 1 {
h.sigT.FailNow()
} else if v == 2 {
h.sigT.SkipNow()
}
case <-time.After(time.Duration(waitSecs) * time.Second):
cb(false)
<-h.sigCh
h.sigT.FailNow()
}
}
func (h *TestSubHarnessBase) VerifyOpSent(err error) {
if err != nil {
h.sigT.Fatal(err.Error())
return
}
}
type TestSubHarness struct {
TestSubHarnessBase
sigOp PendingOp
}
func makeTestSubHarness(t testing.TB) *TestSubHarness {
// Note that the signaling channel here must have a queue of
// at least 1 to avoid deadlocks during cancellations.
h := &TestSubHarness{
TestSubHarnessBase: TestSubHarnessBase{
sigT: t,
sigCh: make(chan int, 1),
},
}
return h
}
func (h *TestSubHarness) Wait(waitSecs int) {
if h.sigOp == nil {
panic("Cannot wait if there is no op set on signaler")
}
h.TestSubHarnessBase.Wait(waitSecs, func(success bool) {
if success {
h.sigOp = nil
return
}
h.sigOp.Cancel()
})
}
func (h *TestSubHarness) PushOp(op PendingOp, err error) {
h.TestSubHarnessBase.VerifyOpSent(err)
if h.sigOp != nil {
panic("Can only set one op on the signaler at a time")
}
h.sigOp = op
}
type TestTxnsSubHarness struct {
TestSubHarnessBase
}
func makeTestTxnsSubHarness(t *testing.T) *TestTxnsSubHarness {
// Note that the signaling channel here must have a queue of
// at least 1 to avoid deadlocks during cancellations.
h := &TestTxnsSubHarness{
TestSubHarnessBase: TestSubHarnessBase{
sigT: t,
sigCh: make(chan int, 1),
},
}
return h
}
func (h *TestTxnsSubHarness) Wait(waitSecs int) {
h.TestSubHarnessBase.Wait(waitSecs, func(success bool) {})
}
gocbcore-10.2.3/testmain_test.go 0000664 0000000 0000000 00000020213 14417540156 0016577 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"crypto/tls"
"crypto/x509"
"flag"
"fmt"
"log"
"os"
"runtime"
"runtime/pprof"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
)
var globalTestLogger *testLogger
type testLogger struct {
Parent Logger
LogCount []uint64
suppressWarnings uint32
}
func (logger *testLogger) Log(level LogLevel, offset int, format string, v ...interface{}) error {
if level >= 0 && level < LogMaxVerbosity {
if atomic.LoadUint32(&logger.suppressWarnings) == 1 && level == LogWarn {
level = LogInfo
}
atomic.AddUint64(&logger.LogCount[level], 1)
}
return logger.Parent.Log(level, offset+1, fmt.Sprintf("[%s] ", logLevelToString(level))+format, v...)
}
func (logger *testLogger) SuppressWarnings(suppress bool) {
if suppress {
atomic.StoreUint32(&logger.suppressWarnings, 1)
} else {
atomic.StoreUint32(&logger.suppressWarnings, 0)
}
}
func createTestLogger() *testLogger {
return &testLogger{
Parent: VerboseStdioLogger(),
LogCount: make([]uint64, LogMaxVerbosity),
}
}
func envFlagString(envName, name, value, usage string) *string {
envValue := os.Getenv(envName)
if envValue != "" {
value = envValue
}
return flag.String(name, value, usage)
}
func envFlagInt(envName, name string, value int, usage string) *int {
envValue := os.Getenv(envName)
if envValue != "" {
var err error
value, err = strconv.Atoi(envValue)
if err != nil {
panic("failed to parse string as int")
}
}
return flag.Int(name, value, usage)
}
func envFlagBool(envName, name string, value bool, usage string) *bool {
envValue := os.Getenv(envName)
if envValue != "" {
if envValue == "0" {
value = false
} else if strings.ToLower(envValue) == "false" {
value = false
} else {
value = true
}
}
return flag.Bool(name, value, usage)
}
func TestMain(m *testing.M) {
initialGoroutineCount := runtime.NumGoroutine()
testSuite := envFlagInt("GCBCTESTSUITE", "test-suite", 1,
"The test suite to run, 1=standard,2=dcp")
connStr := envFlagString("GCBCONNSTR", "connstr", "",
"Connection string to run tests with")
bucketName := envFlagString("GCBBUCKET", "bucket", "default",
"The bucket to use to test against")
disableLogger := envFlagBool("GCBNOLOG", "disable-logger", false,
"Whether or not to disable the logger")
username := envFlagString("GCBUSER", "user", "",
"The username to use to authenticate when using a real server")
password := envFlagString("GCBPASS", "pass", "",
"The password to use to authenticate when using a real server")
clusterVersionStr := envFlagString("GCBCVER", "cluster-version", "7.0.0",
"The server version being tested against (major.minor.patch.build_edition)")
featuresToTest := envFlagString("GCBFEAT", "features", "",
"The features that should be tested")
memdBucketName := envFlagString("GCBMEMDBUCKET", "memd-bucket", "memd",
"The memd bucket to use to test against")
scopeName := envFlagString("GCBSCOPE", "scope-name", "",
"The scope name to use to test with collections")
collectionName := envFlagString("GCBCOLL", "collection-name", "",
"The collection name to use to test with collections")
certsDir := envFlagString("GCBCERTSDIR", "certs-dir", "",
"The path to the directory containing certificates with following names: ca.pem[,client.pem,client.key]")
numMutations := envFlagInt("GCBDCPMUTATIONS", "dcp-num-mutations", 50,
"The number of mutations to create")
numDeletions := envFlagInt("GCBDCPDELETIONS", "dcp-num-deletions", 5,
"The number of deletions to create")
numExpirations := envFlagInt("GCBDCPEXPIRATIONS", "dcp-num-expirations", 5,
"The number of expirations to create")
numScopes := envFlagInt("GCBDCPSCOPES", "dcp-num-scopes", 2,
"The number of scopes to create")
numCollections := envFlagInt("GCBDCPCOLLECTIONS", "dcp-num-colletions", 5,
"The number of collections to create, per scope")
mockPath := envFlagString("GCBMOCKPATH", "mock-path", "",
"Path to the mock, if not using a downloaded build")
flag.Parse()
clusterVersion, err := nodeVersionFromString(*clusterVersionStr)
if err != nil {
panic("failed to parse specified cluster version")
}
featureFlags := ParseFeatureFlags(*featuresToTest)
var authenticator AuthProvider
var caProvider func() *x509.CertPool
if len(*certsDir) > 0 {
ca, cert, err := ParseCerts(*certsDir)
if err != nil {
panic("failed to parse certificates")
}
// Just because we have a root cert doesn't mean that we have client certs.
if cert == nil {
authenticator = &PasswordAuthProvider{
Username: *username,
Password: *password,
}
} else {
authenticator = &CertificateAuthenticator{
ClientCertificate: cert,
}
}
caProvider = func() *x509.CertPool {
return ca
}
} else {
authenticator = &PasswordAuthProvider{
Username: *username,
Password: *password,
}
}
if *testSuite == 1 {
globalTestConfig = &TestConfig{
ConnStr: *connStr,
BucketName: *bucketName,
MemdBucketName: *memdBucketName,
ScopeName: *scopeName,
CollectionName: *collectionName,
Authenticator: authenticator,
CAProvider: caProvider,
ClusterVersion: clusterVersion,
FeatureFlags: featureFlags,
MockPath: *mockPath,
}
} else if *testSuite == 2 {
globalDCPTestConfig = &DCPTestConfig{
ConnStr: *connStr,
BucketName: *bucketName,
Authenticator: authenticator,
CAProvider: caProvider,
ClusterVersion: clusterVersion,
FeatureFlags: featureFlags,
NumMutations: *numMutations,
NumDeletions: *numDeletions,
NumExpirations: *numExpirations,
NumScopes: *numScopes,
NumCollections: *numCollections,
}
} else {
panic("Unrecognized test suite requested")
}
if !*disableLogger {
// Set up our special logger which logs the log level count
globalTestLogger = createTestLogger()
SetLogger(globalTestLogger)
}
result := m.Run()
if globalBenchSuite != nil {
if err := globalBenchSuite.Close(); err != nil {
log.Printf("Failed to close benchmarking suite, failing")
result = 1
}
}
if globalTestLogger != nil {
log.Printf("Log Messages Emitted:")
var preLogTotal uint64
for i := 0; i < int(LogMaxVerbosity); i++ {
count := atomic.LoadUint64(&globalTestLogger.LogCount[i])
preLogTotal += count
log.Printf(" (%s): %d", logLevelToString(LogLevel(i)), count)
}
abnormalLogCount := atomic.LoadUint64(&globalTestLogger.LogCount[LogError]) + atomic.LoadUint64(&globalTestLogger.LogCount[LogWarn])
if abnormalLogCount > 0 {
log.Printf("Detected unexpected logging, failing")
result = 1
}
time.Sleep(1 * time.Second)
log.Printf("Post sleep log Messages Emitted:")
var postLogTotal uint64
for i := 0; i < int(LogMaxVerbosity); i++ {
count := atomic.LoadUint64(&globalTestLogger.LogCount[i])
postLogTotal += count
log.Printf(" (%s): %d", logLevelToString(LogLevel(i)), count)
}
if preLogTotal != postLogTotal {
log.Printf("Detected unexpected logging after agent closed, failing")
result = 1
}
}
// Loop for at most a second checking for goroutines leaks, this gives any HTTP goroutines time to shutdown
start := time.Now()
var finalGoroutineCount int
for time.Since(start) <= 1*time.Second {
runtime.Gosched()
finalGoroutineCount = runtime.NumGoroutine()
if finalGoroutineCount == initialGoroutineCount {
break
}
time.Sleep(10 * time.Millisecond)
}
if finalGoroutineCount != initialGoroutineCount {
log.Printf("Detected a goroutine leak (%d before != %d after), failing", initialGoroutineCount, finalGoroutineCount)
pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
result = 1
} else {
log.Printf("No goroutines appear to have leaked (%d before == %d after)", initialGoroutineCount, finalGoroutineCount)
}
os.Exit(result)
}
type CertificateAuthenticator struct {
ClientCertificate *tls.Certificate
}
func (ca CertificateAuthenticator) SupportsTLS() bool {
return true
}
func (ca CertificateAuthenticator) SupportsNonTLS() bool {
return false
}
func (ca CertificateAuthenticator) Certificate(req AuthCertRequest) (*tls.Certificate, error) {
return ca.ClientCertificate, nil
}
func (ca CertificateAuthenticator) Credentials(req AuthCredsRequest) ([]UserPassPair, error) {
return []UserPassPair{{
Username: "",
Password: "",
}}, nil
}
gocbcore-10.2.3/testnames_test.go 0000664 0000000 0000000 00000000641 14417540156 0016761 0 ustar 00root root 0000000 0000000 package gocbcore
type TestName string
const (
TestNameMemcachedBasic TestName = "kv/memcached/MemcachedBasic"
TestNameErrMapLinearRetry TestName = "kv/errmap/ErrMapLinearRetry"
TestNameErrMapConstantRetry TestName = "kv/errmap/ErrMapConstantRetry"
TestNameErrMapExponentialRetry TestName = "kv/errmap/ErrMapExponentialRetry"
TestNameExtendedError TestName = "kv/error/ExtendedError"
)
gocbcore-10.2.3/teststandardsuite_test.go 0000664 0000000 0000000 00000035510 14417540156 0020533 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"crypto/x509"
"encoding/json"
"fmt"
"github.com/couchbase/gocbcore/v10/memd"
cavescli "github.com/couchbaselabs/gocaves/client"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
"io/ioutil"
"log"
"net/http"
"strings"
"testing"
"time"
)
type StandardTestSuite struct {
suite.Suite
*TestConfig
agentGroup *AgentGroup
mockInst *cavescli.Client
runID string
tracer *testTracer
meter *testMeter
}
func (suite *StandardTestSuite) BeforeTest(suiteName, testName string) {
suite.tracer.Reset()
suite.meter.Reset()
}
func (suite *StandardTestSuite) SetupSuite() {
if globalTestConfig.ConnStr == "" {
suite.mockInst, suite.runID = setupMock(false)
}
suite.TestConfig = globalTestConfig
suite.tracer = newTestTracer()
suite.meter = newTestMeter()
var err error
suite.agentGroup, err = suite.initAgentGroup(suite.makeAgentGroupConfig(globalTestConfig))
suite.Require().Nil(err, err)
err = suite.agentGroup.OpenBucket(globalTestConfig.BucketName)
suite.Require().Nil(err, err)
// If we don't do a wait until ready then it can be difficult to verify tracing behavior on the
// first test that runs.
s := suite.GetHarness()
s.PushOp(suite.DefaultAgent().WaitUntilReady(
time.Now().Add(5*time.Second),
WaitUntilReadyOptions{},
func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitUntilReady operation failed: %v", err)
}
})
}),
)
s.Wait(0)
}
func (suite *StandardTestSuite) TearDownSuite() {
err := suite.agentGroup.Close()
suite.Require().Nil(err, err)
if suite.mockInst != nil {
_, err := suite.mockInst.EndTesting(suite.runID)
if err != nil {
log.Printf("Failed to end testing: %v", err)
}
err = suite.mockInst.Shutdown()
suite.Require().Nil(err, err)
}
}
func (suite *StandardTestSuite) TimeTravel(waitDura time.Duration) {
if suite.mockInst == nil {
time.Sleep(waitDura)
return
}
err := suite.mockInst.TimeTravelRun(suite.runID, waitDura)
suite.Require().Nil(err, err)
}
func (suite *StandardTestSuite) IsMockServer() bool {
return suite.mockInst != nil
}
func (suite *StandardTestSuite) SupportsFeature(feature TestFeatureCode) bool {
featureFlagValue := 0
for _, featureFlag := range suite.FeatureFlags {
if featureFlag.Feature == feature || featureFlag.Feature == "*" {
if featureFlag.Enabled {
featureFlagValue = +1
} else {
featureFlagValue = -1
}
}
}
if featureFlagValue == -1 {
return false
} else if featureFlagValue == +1 {
return true
}
switch feature {
case TestFeatureSsl:
return true
case TestFeatureViews:
return true
case TestFeatureErrMap:
return true
case TestFeatureReplicas:
return true
case TestFeatureMemd:
return true
case TestFeatureN1ql:
return !suite.IsMockServer() && !suite.ClusterVersion.Equal(srvVer650DP)
case TestFeatureCbas:
return !suite.IsMockServer() && suite.ClusterVersion.Higher(srvVer600) &&
!suite.ClusterVersion.Equal(srvVer650DP)
case TestFeatureFts:
return !suite.IsMockServer() && !suite.ClusterVersion.Lower(srvVer551)
case TestFeatureCollections:
return suite.ClusterVersion.Equal(srvVer650DP) || !suite.ClusterVersion.Lower(srvVer700)
case TestFeatureGetMeta:
return !suite.IsMockServer()
case TestFeatureGCCCP:
return !suite.IsMockServer() && !suite.ClusterVersion.Lower(srvVer650)
case TestFeatureEnhancedDurability:
return !suite.ClusterVersion.Lower(srvVer650)
case TestFeatureCreateDeleted:
return !suite.ClusterVersion.Lower(srvVer660)
case TestFeatureReplaceBodyWithXattr:
return !suite.IsMockServer() && !suite.ClusterVersion.Lower(srvVer700)
case TestFeatureExpandMacros:
return !suite.ClusterVersion.Lower(srvVer450)
case TestFeatureExpandMacrosSeqNo:
return !suite.IsMockServer() && !suite.ClusterVersion.Lower(srvVer450)
case TestFeaturePreserveExpiry:
return !suite.IsMockServer() && !suite.ClusterVersion.Lower(srvVer700)
case TestFeatureTransactions:
return !suite.ClusterVersion.Lower(srvVer700)
case TestFeatureN1qlReasons:
return !suite.IsMockServer() && !suite.ClusterVersion.Lower(srvVer710)
case TestFeatureResourceUnits:
return !suite.IsMockServer() && suite.ClusterVersion.Equal(srvVer720DP)
case TestFeatureRangeScan:
return !suite.IsMockServer() && !suite.ClusterVersion.Lower(srvVer750)
}
panic("found unsupported feature code")
}
func (suite *StandardTestSuite) DefaultAgent() *Agent {
return suite.agentGroup.GetAgent(globalTestConfig.BucketName)
}
func (suite *StandardTestSuite) AgentGroup() *AgentGroup {
return suite.agentGroup
}
func (suite *StandardTestSuite) GetHarness() *TestSubHarness {
return makeTestSubHarness(suite.T())
}
func (suite *StandardTestSuite) GetTxnHarness() *TestTxnsSubHarness {
return makeTestTxnsSubHarness(suite.T())
}
func (suite *StandardTestSuite) GetAgentAndHarness() (*Agent, *TestSubHarness) {
h := suite.GetHarness()
return suite.DefaultAgent(), h
}
func (suite *StandardTestSuite) GetAgentAndTxnHarness() (*Agent, *TestTxnsSubHarness) {
h := suite.GetTxnHarness()
return suite.DefaultAgent(), h
}
func (suite *StandardTestSuite) EnsureSupportsFeature(feature TestFeatureCode) {
if !suite.SupportsFeature(feature) {
suite.T().Skipf("Skipping test due to disabled feature code: %s", feature)
}
}
type TestSpec struct {
Agent *Agent
Collection string
Scope string
Tracer *testTracer
Meter *testMeter
}
func (suite *StandardTestSuite) StartTest(name TestName) TestSpec {
var connStr, bucket, scope, collection string
if suite.IsMockServer() {
spec, err := suite.mockInst.StartTest(suite.runID, string(name))
suite.Require().Nil(err)
if spec.ConnStr == "" {
return TestSpec{
Agent: suite.DefaultAgent(),
Collection: globalTestConfig.CollectionName,
Scope: globalTestConfig.ScopeName,
Tracer: suite.tracer,
Meter: suite.meter,
}
}
connStr = spec.ConnStr
bucket = spec.BucketName
scope = spec.ScopeName
collection = spec.CollectionName
} else {
if name != TestNameMemcachedBasic {
return TestSpec{
Agent: suite.DefaultAgent(),
Collection: globalTestConfig.CollectionName,
Scope: globalTestConfig.ScopeName,
Tracer: suite.tracer,
Meter: suite.meter,
}
}
connStr = globalTestConfig.ConnStr
bucket = "memd"
scope = "_default"
collection = "_default"
}
baseCfg := globalTestConfig.Clone()
baseCfg.ConnStr = connStr
tracer := newTestTracer()
meter := newTestMeter()
cfg := makeAgentConfig(baseCfg)
cfg.BucketName = bucket
cfg.TracerConfig.Tracer = tracer
cfg.MeterConfig.Meter = meter
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
// Prime the agent to ensure that operations are clear to send without messing with tracing spans.
s := suite.GetHarness()
s.PushOp(agent.WaitUntilReady(time.Now().Add(5*time.Second), WaitUntilReadyOptions{}, func(result *WaitUntilReadyResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("WaitUntilReady failed with error: %v", err)
}
})
}))
s.Wait(6)
return TestSpec{
Agent: agent,
Scope: scope,
Collection: collection,
Tracer: tracer,
Meter: meter,
}
}
func (suite *StandardTestSuite) EndTest(spec TestSpec) {
agent := spec.Agent
if agent == suite.DefaultAgent() {
return
}
err := agent.Close()
suite.Assert().Nil(err, err)
if suite.IsMockServer() {
err = suite.mockInst.EndTest(suite.runID)
suite.Require().Nil(err, err)
}
}
func (suite *StandardTestSuite) LoadConfigFromFile(filename string) (cfg *cfgBucket) {
s, err := ioutil.ReadFile(filename)
if err != nil {
suite.T().Fatal(err.Error())
}
rawCfg, err := parseConfig(s, "localhost")
if err != nil {
suite.T().Fatal(err.Error())
}
cfg = rawCfg
return
}
func makeAgentConfig(testConfig *TestConfig) AgentConfig {
config := AgentConfig{}
config.FromConnStr(testConfig.ConnStr)
config.IoConfig = IoConfig{
UseDurations: true,
UseMutationTokens: true,
UseCollections: true,
UseOutOfOrderResponses: true,
}
config.SecurityConfig.Auth = testConfig.Authenticator
if testConfig.CAProvider != nil {
config.SecurityConfig.TLSRootCAProvider = testConfig.CAProvider
}
return config
}
func (suite *StandardTestSuite) makeAgentGroupConfig(testConfig *TestConfig) AgentGroupConfig {
config := AgentGroupConfig{}
config.FromConnStr(testConfig.ConnStr)
config.IoConfig = IoConfig{
UseDurations: true,
UseMutationTokens: true,
UseCollections: true,
UseOutOfOrderResponses: true,
}
config.TracerConfig.Tracer = suite.tracer
config.MeterConfig.Meter = suite.meter
config.InternalConfig.EnableResourceUnitsTrackingHello = true
config.SecurityConfig.Auth = testConfig.Authenticator
if config.SecurityConfig.UseTLS {
if testConfig.CAProvider == nil {
config.SecurityConfig.TLSRootCAProvider = func() *x509.CertPool {
return nil
}
} else {
config.SecurityConfig.TLSRootCAProvider = testConfig.CAProvider
}
}
return config
}
func (suite *StandardTestSuite) initAgentGroup(config AgentGroupConfig) (*AgentGroup, error) {
ag, err := CreateAgentGroup(&config)
if err != nil {
return nil, err
}
return ag, nil
}
func (suite *StandardTestSuite) tryAtMost(times int, interval time.Duration, fn func() bool) bool {
i := 0
for {
success := fn()
if success {
return true
}
i++
if i >= times {
return false
}
time.Sleep(interval)
}
}
func (suite *StandardTestSuite) tryUntil(deadline time.Time, interval time.Duration, fn func() bool) bool {
for {
success := fn()
if success {
return true
}
sleepDeadline := time.Now().Add(interval)
if sleepDeadline.After(deadline) {
return false
}
time.Sleep(sleepDeadline.Sub(time.Now()))
}
}
func (suite *StandardTestSuite) mustMarshal(content interface{}) []byte {
b, err := json.Marshal(content)
suite.Require().Nil(err, err)
return b
}
func (suite *StandardTestSuite) mustSetDoc(agent *Agent, s *TestSubHarness, key []byte, content interface{}) (casOut Cas) {
s.PushOp(agent.Set(SetOptions{
Key: key,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
Value: suite.mustMarshal(content),
}, func(result *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Expected error to be nil but was %v", err)
}
if result.Cas == 0 {
s.Fatalf("Expected cas to be non 0")
}
casOut = result.Cas
})
}))
s.Wait(0)
return
}
func (suite *StandardTestSuite) mustGetDoc(agent *Agent, s *TestSubHarness, key []byte) (valOut []byte, casOut Cas) {
s.PushOp(agent.Get(GetOptions{
Key: key,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
}, func(result *GetResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Expected error to be nil but was %v", err)
}
valOut = result.Value
casOut = result.Cas
})
}))
s.Wait(0)
return
}
func (suite *StandardTestSuite) lookupDoc(agent *Agent, s *TestSubHarness, ops []SubDocOp,
key []byte) (valOut *LookupInResult, errOut error) {
s.PushOp(agent.LookupIn(LookupInOptions{
Key: key,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
Ops: ops,
}, func(result *LookupInResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Expected error to be nil but was %v", err)
}
valOut = result
})
}))
s.Wait(0)
return
}
func (suite *StandardTestSuite) mutateIn(agent *Agent, s *TestSubHarness, ops []SubDocOp, key []byte,
cas Cas, flags memd.SubdocDocFlag) (casOut Cas, errOut error) {
s.PushOp(agent.MutateIn(MutateInOptions{
Key: key,
Cas: cas,
Ops: ops,
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
Flags: flags,
}, func(result *MutateInResult, err error) {
s.Wrap(func() {
if err != nil {
errOut = err
return
}
casOut = result.Cas
})
}))
s.Wait(0)
return
}
func (suite *StandardTestSuite) CreateNSAgentConfig() (*AgentConfig, string) {
defaultAgent := suite.DefaultAgent()
snapshot, err := defaultAgent.kvMux.PipelineSnapshot()
suite.Require().Nil(err, err)
if snapshot.NumPipelines() == 1 {
suite.T().Skip("Skipping test due to cluster only containing one node")
}
srcCfg := makeAgentConfig(globalTestConfig)
if len(srcCfg.SeedConfig.HTTPAddrs) == 0 {
suite.T().Skip("Skipping test due to no HTTP addresses")
}
seedAddr := srcCfg.SeedConfig.HTTPAddrs[0]
parts := strings.Split(seedAddr, ":")
if parts[1] != "8091" && parts[1] != "11210" {
// This should work with non default ports but it makes the test logic too complicated.
// This implicitly means that if TLS is enabled then this test won't run.
suite.T().Skip("Skipping test due to non default ports have been supplied")
}
connstr := fmt.Sprintf("ns_server://%s", seedAddr)
config := &AgentConfig{}
err = config.FromConnStr(connstr)
suite.Require().Nil(err, err)
config.IoConfig = IoConfig{
UseDurations: true,
UseMutationTokens: true,
UseCollections: true,
UseOutOfOrderResponses: true,
}
config.SecurityConfig.Auth = globalTestConfig.Authenticator
config.SecurityConfig.UseTLS = true
config.SecurityConfig.TLSRootCAProvider = func() *x509.CertPool {
return nil
}
config.BucketName = globalTestConfig.BucketName
return config, seedAddr
}
func (suite *StandardTestSuite) VerifyKVMetrics(meter *testMeter, operation string, num int, atLeastNum bool, zeroLenAllowed bool) {
suite.VerifyMetrics(meter, makeMetricsKey("kv", operation), num, atLeastNum, zeroLenAllowed)
}
func (suite *StandardTestSuite) VerifyMetrics(meter *testMeter, key string, num int, atLeastNum bool, zeroLenAllowed bool) {
meter.lock.Lock()
defer meter.lock.Unlock()
recorders := meter.recorders
if suite.Assert().Contains(recorders, key) {
if atLeastNum {
suite.Assert().GreaterOrEqual(len(recorders[key].values), num)
} else {
suite.Assert().Len(recorders[key].values, num)
}
for _, val := range recorders[key].values {
if !zeroLenAllowed {
suite.Assert().NotZero(val)
}
}
}
}
func setupMock(quiet bool) (*cavescli.Client, string) {
m, err := cavescli.NewClient(cavescli.NewClientOptions{
Version: "v0.0.1-75",
Quiet: quiet,
})
if err != nil {
panic(err)
}
runID := uuid.New().String()
connstr, err := m.StartTesting(runID, "gocbcore-"+Version())
if err != nil {
panic(err)
}
globalTestConfig.ConnStr = connstr
globalTestConfig.BucketName = "default"
globalTestConfig.MemdBucketName = "memd"
globalTestConfig.Authenticator = &PasswordAuthProvider{
Username: "Administrator",
Password: "password",
}
// gocbcore itself doesn't use the default client but the mock downloader does so let's make sure that it
// doesn't hold any goroutines open which will affect our goroutine leak detector.
http.DefaultClient.CloseIdleConnections()
return m, runID
}
func TestStandardSuite(t *testing.T) {
if globalTestConfig == nil {
t.Skip()
}
suite.Run(t, new(StandardTestSuite))
}
gocbcore-10.2.3/testunitsuite_test.go 0000664 0000000 0000000 00000000777 14417540156 0017721 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"io/ioutil"
"testing"
"github.com/stretchr/testify/suite"
)
type UnitTestSuite struct {
suite.Suite
}
func TestUnitSuite(t *testing.T) {
if globalTestConfig == nil {
t.Skip()
}
suite.Run(t, new(UnitTestSuite))
}
func (suite *UnitTestSuite) LoadRawTestDataset(dataset string) ([]byte, error) {
return ioutil.ReadFile("testdata/" + dataset + ".json")
}
func loadRawTestDataset(dataset string) ([]byte, error) {
return ioutil.ReadFile("testdata/" + dataset + ".json")
}
gocbcore-10.2.3/timerpool.go 0000664 0000000 0000000 00000001221 14417540156 0015724 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"sync"
"time"
)
var globalTimerPool sync.Pool
// AcquireTimer acquires a time from a global pool of timers maintained by the library.
func AcquireTimer(d time.Duration) *time.Timer {
tmr, isTmr := globalTimerPool.Get().(*time.Timer)
if tmr == nil || !isTmr {
if !isTmr && tmr != nil {
logErrorf("Encountered non-timer in timer pool")
}
return time.NewTimer(d)
}
tmr.Reset(d)
return tmr
}
// ReleaseTimer returns a timer to the global pool of timers maintained by the library.
func ReleaseTimer(t *time.Timer, wasRead bool) {
stopped := t.Stop()
if !wasRead && !stopped {
<-t.C
}
globalTimerPool.Put(t)
}
gocbcore-10.2.3/tracing.go 0000664 0000000 0000000 00000017311 14417540156 0015350 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"net"
"net/http"
"strconv"
"sync"
"time"
)
// RequestTracer describes the tracing abstraction in the SDK.
type RequestTracer interface {
RequestSpan(parentContext RequestSpanContext, operationName string) RequestSpan
}
// RequestSpan is the interface for spans that are created by a RequestTracer.
type RequestSpan interface {
End()
Context() RequestSpanContext
AddEvent(name string, timestamp time.Time)
SetAttribute(key string, value interface{})
}
// RequestSpanContext is the interface for for external span contexts that can be passed in into the SDK option blocks.
type RequestSpanContext interface {
}
type noopSpan struct{}
type noopSpanContext struct{}
var (
defaultNoopSpanContext = noopSpanContext{}
defaultNoopSpan = noopSpan{}
)
type noopTracer struct {
}
func (tracer noopTracer) RequestSpan(parentContext RequestSpanContext, operationName string) RequestSpan {
return defaultNoopSpan
}
func (span noopSpan) End() {
}
func (span noopSpan) Context() RequestSpanContext {
return defaultNoopSpanContext
}
func (span noopSpan) SetAttribute(key string, value interface{}) {
}
func (span noopSpan) AddEvent(key string, timestamp time.Time) {
}
type opTracer struct {
parentContext RequestSpanContext
opSpan RequestSpan
}
func (tracer *opTracer) Finish() {
if tracer.opSpan != nil {
tracer.opSpan.End()
}
}
func (tracer *opTracer) RootContext() RequestSpanContext {
if tracer.opSpan != nil {
return tracer.opSpan.Context()
}
return tracer.parentContext
}
type tracerComponent struct {
tracer RequestTracer
bucket string
noRootTraceSpans bool
metrics Meter
valueRecorderAttribsCache sync.Map
}
func newTracerComponent(tracer RequestTracer, bucket string, noRootTraceSpans bool, metrics Meter) *tracerComponent {
return &tracerComponent{
tracer: tracer,
bucket: bucket,
noRootTraceSpans: noRootTraceSpans,
metrics: metrics,
}
}
func (tc *tracerComponent) CreateOpTrace(operationName string, parentContext RequestSpanContext) *opTracer {
if tc.noRootTraceSpans {
return &opTracer{
parentContext: parentContext,
opSpan: nil,
}
}
opSpan := tc.tracer.RequestSpan(parentContext, operationName)
opSpan.SetAttribute(spanAttribDBSystemKey, spanAttribDBSystemValue)
return &opTracer{
parentContext: parentContext,
opSpan: opSpan,
}
}
func (tc *tracerComponent) StartHTTPDispatchSpan(req *httpRequest, name string) RequestSpan {
span := tc.tracer.RequestSpan(req.RootTraceContext, name)
return span
}
func (tc *tracerComponent) StopHTTPDispatchSpan(span RequestSpan, req *http.Request, id string, retries uint32) {
span.SetAttribute(spanAttribDBSystemKey, spanAttribDBSystemValue)
span.SetAttribute(spanAttribNetTransportKey, spanAttribNetTransportValue)
if id != "" {
span.SetAttribute(spanAttribOperationIDKey, id)
}
remoteName, remotePort, err := net.SplitHostPort(req.Host)
if err != nil {
logDebugf("Failed to split host port: %s", err)
}
span.SetAttribute(spanAttribNetPeerNameKey, remoteName)
span.SetAttribute(spanAttribNetPeerPortKey, remotePort)
span.SetAttribute(spanAttribNumRetries, retries)
span.End()
}
func (tc *tracerComponent) StartCmdTrace(req *memdQRequest) {
if req.cmdTraceSpan != nil {
logWarnf("Attempted to start tracing on traced request OP=0x%x, Opaque=%d", req.Command, req.Opaque)
return
}
if req.RootTraceContext == nil {
return
}
req.processingLock.Lock()
req.cmdTraceSpan = tc.tracer.RequestSpan(req.RootTraceContext, req.Packet.Command.Name())
req.processingLock.Unlock()
}
func (tc *tracerComponent) StartNetTrace(req *memdQRequest) {
req.processingLock.Lock()
if req.cmdTraceSpan == nil {
req.processingLock.Unlock()
return
}
if req.netTraceSpan != nil {
req.processingLock.Unlock()
logWarnf("Attempted to start net tracing on traced request")
return
}
req.netTraceSpan = tc.tracer.RequestSpan(req.cmdTraceSpan.Context(), spanNameDispatchToServer)
req.processingLock.Unlock()
}
func (tc *tracerComponent) ResponseValueRecord(service, operation string, start time.Time) {
if tc.metrics == nil {
return
}
key := service + "." + operation
attribs, ok := tc.valueRecorderAttribsCache.Load(key)
if !ok {
// It doesn't really matter if we end up storing the attribs against the same key multiple times. We just need
// to have a read efficient cache that doesn't cause actual data races.
attribs = map[string]string{
metricAttribServiceKey: service,
}
if operation != "" {
attribs.(map[string]string)[metricAttribOperationKey] = operation
}
tc.valueRecorderAttribsCache.Store(key, attribs)
}
recorder, err := tc.metrics.ValueRecorder(meterNameCBOperations, attribs.(map[string]string))
if err != nil {
logDebugf("Failed to get value recorder: %v", err)
return
}
duration := uint64(time.Since(start).Microseconds())
if duration == 0 {
duration = uint64(1 * time.Microsecond)
}
recorder.RecordValue(duration)
}
func stopCmdTrace(req *memdQRequest) {
if req.RootTraceContext == nil {
return
}
if req.cmdTraceSpan == nil {
logWarnf("Attempted to stop tracing on untraced request")
return
}
req.cmdTraceSpan.SetAttribute(spanAttribDBSystemKey, "couchbase")
req.cmdTraceSpan.SetAttribute(spanAttribNumRetries, req.RetryAttempts())
req.cmdTraceSpan.End()
req.cmdTraceSpan = nil
}
func cancelReqTrace(req *memdQRequest, local, remote string) {
if req.cmdTraceSpan != nil {
if req.netTraceSpan != nil {
stopNetTrace(req, nil, local, remote)
}
stopCmdTrace(req)
}
}
func stopNetTrace(req *memdQRequest, resp *memdQResponse, localAddress, remoteAddress string) {
if req.cmdTraceSpan == nil {
return
}
if req.netTraceSpan == nil {
logWarnf("Attempted to stop net tracing on an untraced request")
return
}
req.netTraceSpan.SetAttribute(spanAttribDBSystemKey, spanAttribDBSystemValue)
req.netTraceSpan.SetAttribute(spanAttribNetTransportKey, spanAttribNetTransportValue)
if resp != nil {
req.netTraceSpan.SetAttribute(spanAttribOperationIDKey, strconv.Itoa(int(resp.Opaque)))
req.netTraceSpan.SetAttribute(spanAttribLocalIDKey, resp.sourceConnID)
}
localName, localPort, err := net.SplitHostPort(localAddress)
if err != nil {
logDebugf("Failed to split host port: %s", err)
}
remoteName, remotePort, err := net.SplitHostPort(remoteAddress)
if err != nil {
logDebugf("Failed to split host port: %s", err)
}
req.netTraceSpan.SetAttribute(spanAttribNetHostNameKey, localName)
req.netTraceSpan.SetAttribute(spanAttribNetHostPortKey, localPort)
req.netTraceSpan.SetAttribute(spanAttribNetPeerNameKey, remoteName)
req.netTraceSpan.SetAttribute(spanAttribNetPeerPortKey, remotePort)
if resp != nil && resp.Packet.ServerDurationFrame != nil {
req.netTraceSpan.SetAttribute(spanAttribServerDurationKey, resp.Packet.ServerDurationFrame.ServerDuration)
}
req.netTraceSpan.End()
req.netTraceSpan = nil
}
type opTelemetryHandler struct {
tracer *opTracer
service string
operation string
start time.Time
metricsCompleteFn func(string, string, time.Time)
}
func (tc *tracerComponent) StartTelemeteryHandler(service, operation string, traceContext RequestSpanContext) *opTelemetryHandler {
return &opTelemetryHandler{
tracer: tc.CreateOpTrace(operation, traceContext),
service: service,
operation: operation,
start: time.Now(),
metricsCompleteFn: tc.ResponseValueRecord,
}
}
func (oth *opTelemetryHandler) RootContext() RequestSpanContext {
return oth.tracer.RootContext()
}
func (oth *opTelemetryHandler) StartTime() time.Time {
return oth.start
}
func (oth *opTelemetryHandler) Finish() {
oth.tracer.Finish()
oth.metricsCompleteFn(oth.service, oth.operation, oth.start)
}
gocbcore-10.2.3/tracing_test.go 0000664 0000000 0000000 00000017036 14417540156 0016413 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"github.com/couchbase/gocbcore/v10/memd"
"time"
)
type testSpan struct {
Name string
Tags map[string]interface{}
Finished bool
ParentContext RequestSpanContext
Spans map[RequestSpanContext][]*testSpan
}
func (ts *testSpan) End() {
ts.Finished = true
}
func (ts *testSpan) Context() RequestSpanContext {
return ts.Spans
}
func newTestSpan(operationName string, parentContext RequestSpanContext) *testSpan {
return &testSpan{
Name: operationName,
Tags: make(map[string]interface{}),
ParentContext: parentContext,
Spans: make(map[RequestSpanContext][]*testSpan),
}
}
func (ts *testSpan) SetAttribute(key string, value interface{}) {
ts.Tags[key] = value
}
func (ts *testSpan) AddEvent(key string, timestamp time.Time) {
}
type testTracer struct {
Spans map[RequestSpanContext][]*testSpan
}
func newTestTracer() *testTracer {
return &testTracer{
Spans: make(map[RequestSpanContext][]*testSpan),
}
}
func (tt *testTracer) RequestSpan(parentContext RequestSpanContext, operationName string) RequestSpan {
// CCCP looper will send us spans which will mess with our trace verifications.
if operationName == memd.CmdGetClusterConfig.Name() || (operationName == spanNameDispatchToServer && parentContext == nil) {
return &noopSpan{}
}
span := newTestSpan(operationName, parentContext)
if parentContext == nil {
tt.Spans[parentContext] = append(tt.Spans[parentContext], span)
} else {
ctx, ok := parentContext.(map[RequestSpanContext][]*testSpan)
if ok {
ctx[operationName] = append(ctx[operationName], span)
} else {
tt.Spans[parentContext] = append(tt.Spans[parentContext], span)
}
}
return span
}
func (tt *testTracer) Reset() {
tt.Spans = make(map[RequestSpanContext][]*testSpan)
}
func (suite *StandardTestSuite) AssertOpSpan(span *testSpan, expectedName, bucketName, cmdName string, numCmdSpans int,
atLeastNumCmdSpans bool, docID string) {
suite.AssertTopLevelSpan(span, expectedName, bucketName)
if atLeastNumCmdSpans {
suite.AssertCmdSpansGE(span.Spans, cmdName, numCmdSpans, docID)
} else {
suite.AssertCmdSpansEq(span.Spans, cmdName, numCmdSpans, docID)
}
}
func (suite *StandardTestSuite) AssertTopLevelSpan(span *testSpan, expectedName, bucketName string) {
suite.Assert().Equal(expectedName, span.Name)
suite.Assert().Equal(1, len(span.Tags))
suite.Assert().Equal("couchbase", span.Tags["db.system"])
suite.Assert().True(span.Finished)
}
func (suite *StandardTestSuite) AssertCmdSpansEq(parents map[RequestSpanContext][]*testSpan, cmdName string,
num int, docID string) {
spans := parents[cmdName]
if suite.Assert().Equal(num, len(spans)) {
for i := 0; i < num; i++ {
suite.AssertCmdSpan(spans[i], cmdName)
}
}
}
func (suite *StandardTestSuite) AssertCmdSpansGE(parents map[RequestSpanContext][]*testSpan, cmdName string,
num int, docID string) {
spans := parents[cmdName]
if suite.Assert().GreaterOrEqual(num, len(spans)) {
for i := 0; i < len(spans); i++ {
suite.AssertCmdSpan(spans[i], cmdName)
}
}
}
func (suite *StandardTestSuite) AssertCmdSpan(span *testSpan, expectedName string) {
suite.Assert().Equal(expectedName, span.Name)
suite.Assert().Equal(2, len(span.Tags))
suite.Assert().True(span.Finished)
suite.Assert().Equal("couchbase", span.Tags["db.system"])
suite.Assert().Contains(span.Tags, "db.couchbase.retries")
suite.AssertNetSpansEq(span.Spans, 1)
}
func (suite *StandardTestSuite) AssertNetSpansEq(parents map[RequestSpanContext][]*testSpan, num int) {
spans := parents[spanNameDispatchToServer]
if suite.Assert().Equal(num, num) {
for i := 0; i < len(spans); i++ {
suite.AssertNetSpan(spans[i])
}
}
}
func (suite *StandardTestSuite) AssertNetSpan(span *testSpan) {
suite.Assert().Equal(spanNameDispatchToServer, span.Name)
numTags := 8
if duration, ok := span.Tags["db.couchbase.server_duration"]; ok {
suite.Assert().NotZero(duration)
numTags++
}
suite.Assert().Equal(numTags, len(span.Tags))
suite.Assert().True(span.Finished)
suite.Assert().Equal("couchbase", span.Tags["db.system"])
suite.Assert().Equal("IP.TCP", span.Tags["net.transport"])
suite.Assert().NotEmpty(span.Tags["db.couchbase.operation_id"])
suite.Assert().NotEmpty(span.Tags["db.couchbase.local_id"])
suite.Assert().NotEmpty(span.Tags["net.host.name"])
suite.Assert().NotEmpty(span.Tags["net.host.port"])
suite.Assert().NotEmpty(span.Tags["net.peer.name"])
suite.Assert().NotEmpty(span.Tags["net.peer.port"])
}
func (suite *StandardTestSuite) AssertHTTPSpan(span *testSpan, expectedName string) {
suite.Assert().Equal(expectedName, span.Name)
suite.Assert().Equal(1, len(span.Tags))
suite.Assert().Equal("couchbase", span.Tags["db.system"])
suite.Assert().True(span.Finished)
childSpans := span.Spans[spanNameDispatchToServer]
suite.Require().GreaterOrEqual(len(childSpans), 1)
dispatchSpan := childSpans[0]
suite.Assert().Equal(6, len(dispatchSpan.Tags))
suite.Assert().True(dispatchSpan.Finished)
suite.Assert().Equal("couchbase", dispatchSpan.Tags["db.system"])
suite.Assert().Equal("IP.TCP", dispatchSpan.Tags["net.transport"])
suite.Assert().NotEmpty(dispatchSpan.Tags["db.couchbase.operation_id"])
suite.Assert().NotEmpty(dispatchSpan.Tags["net.peer.name"])
suite.Assert().NotEmpty(dispatchSpan.Tags["net.peer.port"])
suite.Assert().Contains(dispatchSpan.Tags, "db.couchbase.retries")
}
func (suite *StandardTestSuite) TestBasicOpsTracingParentNoRoot() {
cfg := makeAgentConfig(globalTestConfig)
cfg.BucketName = globalTestConfig.BucketName
cfg.TracerConfig.NoRootTraceSpans = true
tracer := newTestTracer()
cfg.TracerConfig.Tracer = tracer
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestBasicOpsTracingParentNoRoot", suite.CollectionName, suite.ScopeName)
// Set
s.PushOp(agent.Set(SetOptions{
Key: []byte("testtracerparentnoroot"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
TraceContext: "set_parent",
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(tracer.Spans, "set_parent") {
parents := tracer.Spans["set_parent"]
if suite.Assert().Equal(1, len(parents)) {
suite.AssertCmdSpan(parents[0], memd.CmdSet.Name())
}
}
}
func (suite *StandardTestSuite) TestBasicOpsTracingParentRoot() {
cfg := makeAgentConfig(globalTestConfig)
cfg.BucketName = globalTestConfig.BucketName
tracer := newTestTracer()
cfg.TracerConfig.Tracer = tracer
agent, err := CreateAgent(&cfg)
suite.Require().Nil(err, err)
defer agent.Close()
s := suite.GetHarness()
suite.VerifyConnectedToBucket(agent, s, "TestBasicOpsTracingParentRoot", suite.CollectionName, suite.ScopeName)
// Set
s.PushOp(agent.Set(SetOptions{
Key: []byte("testtracerparentroot"),
Value: []byte("{}"),
CollectionName: suite.CollectionName,
ScopeName: suite.ScopeName,
TraceContext: "set_parent",
}, func(res *StoreResult, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("Set operation failed: %v", err)
}
if res.Cas == Cas(0) {
s.Fatalf("Invalid cas received")
}
})
}))
s.Wait(0)
if suite.Assert().Contains(tracer.Spans, "set_parent") {
parents := tracer.Spans["set_parent"]
if suite.Assert().Equal(1, len(parents)) {
suite.AssertOpSpan(parents[0], "Set", agent.BucketName(), memd.CmdSet.Name(), 1, false, "test")
}
}
}
gocbcore-10.2.3/transaction.go 0000664 0000000 0000000 00000034521 14417540156 0016250 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"time"
"github.com/google/uuid"
)
type addCleanupRequest func(req *TransactionsCleanupRequest) bool
type addLostCleanupLocation func(bucket, scope, collection string)
// Transaction represents a single active transaction, it can be used to
// stage mutations and finally commit them.
type Transaction struct {
parent *TransactionsManager
expiryTime time.Time
startTime time.Time
keyValueTimeout time.Duration
durabilityLevel TransactionDurabilityLevel
enableParallelUnstaging bool
enableNonFatalGets bool
enableExplicitATRs bool
enableMutationCaching bool
atrLocation TransactionATRLocation
bucketAgentProvider TransactionsBucketAgentProviderFn
transactionID string
attempt *transactionAttempt
hooks TransactionHooks
addCleanupRequest addCleanupRequest
addLostCleanupLocation addLostCleanupLocation
recordResourceUnit resourceUnitCallback
logger *internalTransactionLogWrapper
}
// ID returns the transaction ID of this transaction.
func (t *Transaction) ID() string {
return t.transactionID
}
// Attempt returns meta-data about the current attempt to complete the transaction.
func (t *Transaction) Attempt() TransactionAttempt {
if t.attempt == nil {
return TransactionAttempt{}
}
return t.attempt.State()
}
// NewAttempt begins a new attempt with this transaction.
func (t *Transaction) NewAttempt() error {
attemptUUID := uuid.New().String()
t.attempt = &transactionAttempt{
expiryTime: t.expiryTime,
txnStartTime: t.startTime,
keyValueTimeout: t.keyValueTimeout,
durabilityLevel: t.durabilityLevel,
transactionID: t.transactionID,
enableNonFatalGets: t.enableNonFatalGets,
enableParallelUnstaging: t.enableParallelUnstaging,
enableMutationCaching: t.enableMutationCaching,
enableExplicitATRs: t.enableExplicitATRs,
atrLocation: t.atrLocation,
bucketAgentProvider: t.bucketAgentProvider,
id: attemptUUID,
state: TransactionAttemptStateNothingWritten,
stagedMutations: nil,
atrAgent: nil,
atrScopeName: "",
atrCollectionName: "",
atrKey: nil,
hooks: t.hooks,
addCleanupRequest: t.addCleanupRequest,
addLostCleanupLocation: t.addLostCleanupLocation,
logger: t.logger,
recordResourceUnit: t.recordResourceUnit,
}
return nil
}
func (t *Transaction) resumeAttempt(txnData *jsonSerializedAttempt) error {
if txnData.ID.Attempt == "" {
return errors.New("invalid txn data - no attempt id")
}
attemptUUID := txnData.ID.Attempt
var txnState TransactionAttemptState
var atrAgent *Agent
var atrOboUser string
var atrScope, atrCollection string
var atrKey []byte
if txnData.ATR.ID != "" {
// ATR references the specific ATR for this transaction.
if txnData.ATR.Bucket == "" {
return errors.New("invalid atr data - no bucket")
}
foundAtrAgent, foundAtrOboUser, err := t.parent.config.BucketAgentProvider(txnData.ATR.Bucket)
if err != nil {
return err
}
txnState = TransactionAttemptStatePending
atrAgent = foundAtrAgent
atrOboUser = foundAtrOboUser
atrScope = txnData.ATR.Scope
atrCollection = txnData.ATR.Collection
atrKey = []byte(txnData.ATR.ID)
} else {
// No ATR information means its pending with no custom.
txnState = TransactionAttemptStateNothingWritten
atrAgent = nil
atrOboUser = ""
atrScope = ""
atrCollection = ""
atrKey = nil
}
stagedMutations := make([]*transactionStagedMutation, len(txnData.Mutations))
for mutationIdx, mutationData := range txnData.Mutations {
if mutationData.Bucket == "" {
return errors.New("invalid staged mutation - no bucket")
}
if mutationData.ID == "" {
return errors.New("invalid staged mutation - no key")
}
if mutationData.Cas == "" {
return errors.New("invalid staged mutation - no cas")
}
if mutationData.Type == "" {
return errors.New("invalid staged mutation - no type")
}
agent, oboUser, err := t.parent.config.BucketAgentProvider(mutationData.Bucket)
if err != nil {
return err
}
cas, err := strconv.ParseUint(mutationData.Cas, 10, 64)
if err != nil {
return err
}
opType, err := transactionStagedMutationTypeFromString(mutationData.Type)
if err != nil {
return err
}
stagedMutations[mutationIdx] = &transactionStagedMutation{
OpType: opType,
Agent: agent,
OboUser: oboUser,
ScopeName: mutationData.Scope,
CollectionName: mutationData.Collection,
Key: []byte(mutationData.ID),
Cas: Cas(cas),
Staged: nil,
}
}
t.attempt = &transactionAttempt{
expiryTime: t.expiryTime,
txnStartTime: t.startTime,
keyValueTimeout: t.keyValueTimeout,
durabilityLevel: t.durabilityLevel,
transactionID: t.transactionID,
enableNonFatalGets: t.enableNonFatalGets,
enableParallelUnstaging: t.enableParallelUnstaging,
enableMutationCaching: t.enableMutationCaching,
enableExplicitATRs: t.enableExplicitATRs,
atrLocation: t.atrLocation,
bucketAgentProvider: t.bucketAgentProvider,
id: attemptUUID,
state: txnState,
stagedMutations: stagedMutations,
atrAgent: atrAgent,
atrOboUser: atrOboUser,
atrScopeName: atrScope,
atrCollectionName: atrCollection,
atrKey: atrKey,
hooks: t.hooks,
addCleanupRequest: t.addCleanupRequest,
addLostCleanupLocation: t.addLostCleanupLocation,
logger: t.logger,
recordResourceUnit: t.recordResourceUnit,
}
return nil
}
// TransactionGetOptions provides options for a Get operation.
type TransactionGetOptions struct {
Agent *Agent
OboUser string
ScopeName string
CollectionName string
Key []byte
// NoRYOW will disable the RYOW logic used to enable transactions
// to naturally read any mutations they have performed.
// VOLATILE: This parameter is subject to change.
NoRYOW bool
}
// TransactionMutableItemMetaATR represents the ATR for meta.
type TransactionMutableItemMetaATR struct {
BucketName string `json:"bkt"`
ScopeName string `json:"scp"`
CollectionName string `json:"coll"`
DocID string `json:"key"`
}
// TransactionMutableItemMeta represents all the meta-data for a fetched
// item. Most of this is used for later mutation operations.
type TransactionMutableItemMeta struct {
TransactionID string `json:"txn"`
AttemptID string `json:"atmpt"`
ATR TransactionMutableItemMetaATR `json:"atr"`
ForwardCompat map[string][]TransactionForwardCompatibilityEntry `json:"fc,omitempty"`
}
// TransactionGetResult represents the result of a Get or GetOptional operation.
type TransactionGetResult struct {
agent *Agent
oboUser string
scopeName string
collectionName string
key []byte
Meta *TransactionMutableItemMeta
Value []byte
Cas Cas
}
// TransactionGetCallback describes a callback for a completed Get or GetOptional operation.
type TransactionGetCallback func(*TransactionGetResult, error)
// Get will attempt to fetch a document, and fail the transaction if it does not exist.
func (t *Transaction) Get(opts TransactionGetOptions, cb TransactionGetCallback) error {
if t.attempt == nil {
return ErrNoAttempt
}
return t.attempt.Get(opts, cb)
}
// TransactionInsertOptions provides options for a Insert operation.
type TransactionInsertOptions struct {
Agent *Agent
OboUser string
ScopeName string
CollectionName string
Key []byte
Value json.RawMessage
}
// TransactionStoreCallback describes a callback for a completed Replace operation.
type TransactionStoreCallback func(*TransactionGetResult, error)
// Insert will attempt to insert a document.
func (t *Transaction) Insert(opts TransactionInsertOptions, cb TransactionStoreCallback) error {
if t.attempt == nil {
return ErrNoAttempt
}
return t.attempt.Insert(opts, cb)
}
// TransactionReplaceOptions provides options for a Replace operation.
type TransactionReplaceOptions struct {
Document *TransactionGetResult
Value json.RawMessage
}
// Replace will attempt to replace an existing document.
func (t *Transaction) Replace(opts TransactionReplaceOptions, cb TransactionStoreCallback) error {
if t.attempt == nil {
return ErrNoAttempt
}
return t.attempt.Replace(opts, cb)
}
// TransactionRemoveOptions provides options for a Remove operation.
type TransactionRemoveOptions struct {
Document *TransactionGetResult
}
// Remove will attempt to remove a previously fetched document.
func (t *Transaction) Remove(opts TransactionRemoveOptions, cb TransactionStoreCallback) error {
if t.attempt == nil {
return ErrNoAttempt
}
return t.attempt.Remove(opts, cb)
}
// TransactionCommitCallback describes a callback for a completed commit operation.
type TransactionCommitCallback func(error)
// Commit will attempt to commit the transaction, rolling it back and cancelling
// it if it is not capable of doing so.
func (t *Transaction) Commit(cb TransactionCommitCallback) error {
if t.attempt == nil {
return ErrNoAttempt
}
return t.attempt.Commit(cb)
}
// TransactionRollbackCallback describes a callback for a completed rollback operation.
type TransactionRollbackCallback func(error)
// Rollback will attempt to rollback the transaction.
func (t *Transaction) Rollback(cb TransactionRollbackCallback) error {
if t.attempt == nil {
return ErrNoAttempt
}
return t.attempt.Rollback(cb)
}
// HasExpired indicates whether this attempt has expired.
func (t *Transaction) HasExpired() bool {
if t.attempt == nil {
return false
}
return t.attempt.HasExpired()
}
// CanCommit indicates whether this attempt can still be committed.
func (t *Transaction) CanCommit() bool {
if t.attempt == nil {
return false
}
return t.attempt.CanCommit()
}
// ShouldRollback indicates if this attempt should be rolled back.
func (t *Transaction) ShouldRollback() bool {
if t.attempt == nil {
return false
}
return t.attempt.ShouldRollback()
}
// ShouldRetry indicates if this attempt thinks we can retry.
func (t *Transaction) ShouldRetry() bool {
if t.attempt == nil {
return false
}
return t.attempt.ShouldRetry()
}
// FinalErrorToRaise returns the TransactionErrorReason corresponding to the final state of the transaction.
func (t *Transaction) FinalErrorToRaise() TransactionErrorReason {
if t.attempt == nil {
return 0
}
return t.attempt.FinalErrorToRaise()
}
func (t *Transaction) TimeRemaining() time.Duration {
if t.attempt == nil {
return 0
}
return t.attempt.TimeRemaining()
}
// SerializeAttempt will serialize the current transaction attempt, allowing it
// to be resumed later, potentially under a different transactions client. It
// is no longer safe to use this attempt once this has occurred, a new attempt
// must be started to use this object following this call.
func (t *Transaction) SerializeAttempt(cb func([]byte, error)) error {
return t.attempt.Serialize(cb)
}
// GetMutations returns a list of all the current mutations that have been performed
// under this transaction.
func (t *Transaction) GetMutations() []TransactionStagedMutation {
if t.attempt == nil {
return nil
}
return t.attempt.GetMutations()
}
// GetATRLocation returns the ATR location for the current attempt, either by
// identifying where it was placed, or where it will be based on custom atr
// configurations.
func (t *Transaction) GetATRLocation() TransactionATRLocation {
if t.attempt != nil {
return t.attempt.GetATRLocation()
}
return t.atrLocation
}
// SetATRLocation forces the ATR location for the current attempt to a specific
// location. Note that this cannot be called if it has already been set. This
// is currently only safe to call before any mutations have occurred.
func (t *Transaction) SetATRLocation(location TransactionATRLocation) error {
if t.attempt == nil {
return errors.New("cannot set ATR location without an active attempt")
}
return t.attempt.SetATRLocation(location)
}
// Config returns the configured parameters for this transaction.
// Note that the Expiration time is adjusted based on the time left.
// Note also that after a transaction is resumed, the custom atr location
// may no longer reflect the originally configured value.
func (t *Transaction) Config() TransactionOptions {
return TransactionOptions{
CustomATRLocation: t.atrLocation,
ExpirationTime: t.TimeRemaining(),
DurabilityLevel: t.durabilityLevel,
KeyValueTimeout: t.keyValueTimeout,
}
}
// TransactionUpdateStateOptions are the settings available to UpdateState.
// This function must only be called once the transaction has entered query mode.
// Internal: This should never be used and is not supported.
type TransactionUpdateStateOptions struct {
ShouldNotCommit bool
ShouldNotRollback bool
ShouldNotRetry bool
State TransactionAttemptState
Reason TransactionErrorReason
}
func (tuso TransactionUpdateStateOptions) String() string {
return fmt.Sprintf("Should not commit: %t, should not rollback: %t, should not retry: %t, state: %s, reason: %s",
tuso.ShouldNotCommit, tuso.ShouldNotRollback, tuso.ShouldNotRetry, tuso.State, tuso.Reason)
}
// UpdateState will update the internal state of the current attempt.
// Internal: This should never be used and is not supported.
func (t *Transaction) UpdateState(opts TransactionUpdateStateOptions) {
if t.attempt == nil {
return
}
t.attempt.UpdateState(opts)
}
// Logger returns the logger used by this transaction.
// Uncommitted: This API may change in the future.
func (t *Transaction) Logger() TransactionLogger {
return t.logger.wrapped
}
gocbcore-10.2.3/transaction_bench_test.go 0000664 0000000 0000000 00000002754 14417540156 0020451 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"fmt"
"github.com/google/uuid"
"sync/atomic"
"testing"
"time"
)
func BenchmarkTransactionInsertGet(b *testing.B) {
suite := GetBenchSuite()
suite.EnsureSupportsFeature(TestFeatureTransactions, b)
b.ReportAllocs()
agent := suite.GetAgent()
key := []byte(uuid.New().String())
val := []byte(`{"name":"mike"}`)
cfg := &TransactionsConfig{
DurabilityLevel: TransactionDurabilityLevelNone,
BucketAgentProvider: func(bucketName string) (*Agent, string, error) {
// We can always return just this one agent as we only actually
// use a single bucket for this entire test.
return agent, "", nil
},
ExpirationTime: 60 * time.Second,
}
var i uint32
suite.RunParallelTxn(b, cfg, func(txn *Transaction, cb func(error)) error {
keyNum := atomic.AddUint32(&i, 1)
key := []byte(fmt.Sprintf("%s-%d", key, keyNum))
return txn.Insert(TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: key,
Value: val,
}, func(result *TransactionGetResult, err error) {
if err != nil {
cb(err)
return
}
err = txn.Get(TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: key,
}, func(result *TransactionGetResult, err error) {
if err != nil {
cb(err)
return
}
cb(nil)
})
if err != nil {
cb(err)
return
}
})
})
}
gocbcore-10.2.3/transaction_logger.go 0000664 0000000 0000000 00000013746 14417540156 0017615 0 ustar 00root root 0000000 0000000 // nolint: unused
package gocbcore
import (
"fmt"
"log"
"sync"
"time"
)
type loggableDocKey struct {
bucket string
scope string
collection string
key []byte
}
func newLoggableDocKey(bucket, scope, collection string, key []byte) loggableDocKey {
return loggableDocKey{
bucket: bucket,
scope: scope,
collection: collection,
key: key,
}
}
func (rdi loggableDocKey) String() string {
if isLogRedactionLevelFull() || isLogRedactionLevelPartial() {
return redactUserData(rdi.build())
}
return rdi.build()
}
func (rdi loggableDocKey) build() string {
scope := rdi.scope
if scope == "" {
scope = "_default"
}
collection := rdi.collection
if collection == "" {
collection = "_default"
}
return rdi.bucket + "." + scope + "." + collection + "." + string(rdi.key)
}
func (rdi loggableDocKey) redacted() interface{} {
return redactUserData(rdi.build())
}
type loggableATRKey struct {
bucket string
scope string
collection string
key []byte
}
func newLoggableATRKey(bucket, scope, collection string, key []byte) loggableATRKey {
return loggableATRKey{
bucket: bucket,
scope: scope,
collection: collection,
key: key,
}
}
func (rdi loggableATRKey) String() string {
if isLogRedactionLevelFull() {
return redactMetaData(rdi.build())
}
return rdi.build()
}
func (rdi loggableATRKey) build() string {
scope := rdi.scope
if scope == "" {
scope = "_default"
}
collection := rdi.collection
if collection == "" {
collection = "_default"
}
data := rdi.bucket + "." + scope + "." + collection
if len(rdi.key) > 0 {
data = data + "." + string(rdi.key)
}
return data
}
func (rdi loggableATRKey) redacted() interface{} {
return redactMetaData(rdi.build())
}
// TransactionLogger is the logger used for logging in transactions.
// Uncommitted: This API may change in the future.
type TransactionLogger interface {
Log(level LogLevel, offset int, txnID, attemptID, format string, v ...interface{}) error
}
// TransactionLogItem represents an entry in the transaction in memory logging.
type TransactionLogItem struct {
Level LogLevel
args []interface{}
txnID string
attemptID string
timestamp time.Time
fmt string
}
func (item TransactionLogItem) String() string {
return fmt.Sprintf("%s %s/%s %s", item.timestamp.AppendFormat([]byte{}, "15:04:05.000"), item.txnID, item.attemptID, fmt.Sprintf(item.fmt, item.args...))
}
// InMemoryTransactionLogger logs to memory, also logging WARN and ERROR logs to the SDK logger.
// Uncommitted: This API may change in the future.
type InMemoryTransactionLogger struct {
lock sync.Mutex
items []TransactionLogItem
}
// NewInMemoryTransactionLogger returns a new in memory transaction logger.
// Uncommitted: This API may change in the future.
func NewInMemoryTransactionLogger() *InMemoryTransactionLogger {
return &InMemoryTransactionLogger{
items: make([]TransactionLogItem, 0, 256),
}
}
// Logs returns the set of log items created during the transaction.
func (tl *InMemoryTransactionLogger) Logs() []TransactionLogItem {
tl.lock.Lock()
logs := make([]TransactionLogItem, len(tl.items))
copy(logs, tl.items)
tl.lock.Unlock()
return logs
}
// Log logs a new log entry to memory and logs to the SDK logs when the level is WARN or ERROR.
func (tl *InMemoryTransactionLogger) Log(level LogLevel, offset int, txnID, attemptID, fmt string, args ...interface{}) error {
item := TransactionLogItem{
Level: level,
args: args,
txnID: txnID,
attemptID: attemptID,
timestamp: time.Now(),
fmt: fmt,
}
tl.lock.Lock()
tl.items = append(tl.items, item)
tl.lock.Unlock()
if level <= LogWarn {
logExf(level, offset, txnID+"/"+attemptID+" "+fmt, args...)
}
return nil
}
// NoopTransactionLogger logs to the SDK logs when the level is WARN or ERROR.
// Uncommitted: This API may change in the future.
type NoopTransactionLogger struct {
logDirectlyBelowLevel LogLevel
}
// NewNoopTransactionLogger returns a new noop transaction logger.
// Uncommitted: This API may change in the future.
func NewNoopTransactionLogger() *NoopTransactionLogger {
return &NoopTransactionLogger{
logDirectlyBelowLevel: LogInfo,
}
}
// Logs returns an empty slice.
func (n *NoopTransactionLogger) Logs() []TransactionLogItem {
return nil
}
// Log logs to the SDK logs when the level is WARN or ERROR.
func (n *NoopTransactionLogger) Log(level LogLevel, offset int, txnID, attemptID, fmt string, args ...interface{}) error {
if level < n.logDirectlyBelowLevel {
logExf(level, offset, txnID+"/"+attemptID+" "+fmt, args...)
}
return nil
}
type internalTransactionLogWrapper struct {
wrapped TransactionLogger
logDirectlyBelowLevel LogLevel
txnID string
}
func newInternalTransactionLogger(txnID string, wrapped TransactionLogger) *internalTransactionLogWrapper {
return &internalTransactionLogWrapper{
wrapped: wrapped,
logDirectlyBelowLevel: LogInfo,
txnID: txnID[:5],
}
}
func (tl *internalTransactionLogWrapper) logExf(attemptID string, level LogLevel, fmt string, args ...interface{}) {
if attemptID != "" {
attemptID = attemptID[:5]
}
err := tl.wrapped.Log(level, 1, tl.txnID, attemptID, fmt, args...)
if err != nil {
log.Printf("Transaction logger error occurred (%s)\n", err)
}
}
func (tl *internalTransactionLogWrapper) logDebugf(attemptID, format string, v ...interface{}) {
tl.logExf(attemptID, LogDebug, format, v...)
}
func (tl *internalTransactionLogWrapper) logSchedf(attemptID, format string, v ...interface{}) {
tl.logExf(attemptID, LogSched, format, v...)
}
func (tl *internalTransactionLogWrapper) logWarnf(attemptID, format string, v ...interface{}) {
tl.logExf(attemptID, LogWarn, format, v...)
}
func (tl *internalTransactionLogWrapper) logErrorf(attemptID, format string, v ...interface{}) {
tl.logExf(attemptID, LogError, format, v...)
}
func (tl *internalTransactionLogWrapper) logInfof(attemptID, format string, v ...interface{}) {
tl.logExf(attemptID, LogInfo, format, v...)
}
gocbcore-10.2.3/transaction_logger_test.go 0000664 0000000 0000000 00000010411 14417540156 0020636 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"fmt"
"github.com/google/uuid"
)
func (suite *UnitTestSuite) TestTransactionLogger() {
txnID := uuid.NewString()
baseLogger := NewInMemoryTransactionLogger()
logger := newInternalTransactionLogger(txnID, baseLogger)
itemToLog := struct {
SomeItem string
}{"someitem"}
id1 := uuid.NewString()
id2 := uuid.NewString()
id3 := uuid.NewString()
id4 := uuid.NewString()
logger.logDebugf(id1, "encountered issue: %s", "error occurred")
logger.logInfof(id2, "sending request: %d", 112)
logger.logSchedf(id3, "sending request: %v", itemToLog)
logger.logInfof(id4, "%s %d %v", "error occurred", 122, itemToLog)
logs := baseLogger.Logs()
suite.Require().Len(logs, 4)
// Timestamp is 12 chars, plus a space.
suite.Assert().Equal(fmt.Sprintf("%s encountered issue: error occurred", loggerIDShort(txnID, id1)), logs[0].String()[13:])
suite.Assert().Equal(LogDebug, logs[0].Level)
suite.Assert().Equal(fmt.Sprintf("%s sending request: 112", loggerIDShort(txnID, id2)), logs[1].String()[13:])
suite.Assert().Equal(LogInfo, logs[1].Level)
suite.Assert().Equal(fmt.Sprintf("%s sending request: {someitem}", loggerIDShort(txnID, id3)), logs[2].String()[13:])
suite.Assert().Equal(LogSched, logs[2].Level)
suite.Assert().Equal(fmt.Sprintf("%s error occurred 122 {someitem}", loggerIDShort(txnID, id4)), logs[3].String()[13:])
suite.Assert().Equal(LogInfo, logs[3].Level)
}
func (suite *UnitTestSuite) TestTransactionLoggerRedact() {
redacted := newLoggableDocKey("bucket", "scope", "collection", []byte("docID"))
suite.Assert().Equal("bucket.scope.collection.docID", redacted.String())
redacted = newLoggableDocKey("bucket", "", "collection", []byte("docID"))
suite.Assert().Equal("bucket._default.collection.docID", redacted.String())
redacted = newLoggableDocKey("bucket", "scope", "", []byte("docID"))
suite.Assert().Equal("bucket.scope._default.docID", redacted.String())
redacted = newLoggableDocKey("bucket", "", "", []byte("docID"))
suite.Assert().Equal("bucket._default._default.docID", redacted.String())
SetLogRedactionLevel(RedactPartial)
redacted = newLoggableDocKey("bucket", "scope", "collection", []byte("docID"))
suite.Assert().Equal("bucket.scope.collection.docID", redacted.String())
redacted = newLoggableDocKey("bucket", "", "collection", []byte("docID"))
suite.Assert().Equal("bucket._default.collection.docID", redacted.String())
redacted = newLoggableDocKey("bucket", "scope", "", []byte("docID"))
suite.Assert().Equal("bucket.scope._default.docID", redacted.String())
redacted = newLoggableDocKey("bucket", "", "", []byte("docID"))
suite.Assert().Equal("bucket._default._default.docID", redacted.String())
SetLogRedactionLevel(RedactNone)
}
func (suite *UnitTestSuite) TestTransactionLoggerRedactATR() {
redacted := newLoggableATRKey("bucket", "scope", "collection", []byte("_txn:atr-0-#14"))
suite.Assert().Equal("bucket.scope.collection._txn:atr-0-#14", redacted.String())
redacted = newLoggableATRKey("bucket", "", "collection", []byte("_txn:atr-0-#14"))
suite.Assert().Equal("bucket._default.collection._txn:atr-0-#14", redacted.String())
redacted = newLoggableATRKey("bucket", "scope", "", []byte("_txn:atr-0-#14"))
suite.Assert().Equal("bucket.scope._default._txn:atr-0-#14", redacted.String())
redacted = newLoggableATRKey("bucket", "", "", []byte("_txn:atr-0-#14"))
suite.Assert().Equal("bucket._default._default._txn:atr-0-#14", redacted.String())
SetLogRedactionLevel(RedactFull)
redacted = newLoggableATRKey("bucket", "scope", "collection", []byte("_txn:atr-0-#14"))
suite.Assert().Equal("bucket.scope.collection._txn:atr-0-#14", redacted.String())
redacted = newLoggableATRKey("bucket", "", "collection", []byte("_txn:atr-0-#14"))
suite.Assert().Equal("bucket._default.collection._txn:atr-0-#14", redacted.String())
redacted = newLoggableATRKey("bucket", "scope", "", []byte("_txn:atr-0-#14"))
suite.Assert().Equal("bucket.scope._default._txn:atr-0-#14", redacted.String())
redacted = newLoggableATRKey("bucket", "", "", []byte("_txn:atr-0-#14"))
suite.Assert().Equal("bucket._default._default._txn:atr-0-#14", redacted.String())
SetLogRedactionLevel(RedactNone)
}
func loggerIDShort(txnID, attemptID string) string {
return fmt.Sprintf("%s/%s", txnID[:5], attemptID[:5])
}
gocbcore-10.2.3/transaction_resourceunits.go 0000664 0000000 0000000 00000001047 14417540156 0021237 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"errors"
)
type resourceUnitCallback func(result *ResourceUnitResult)
func noopResourceUnitCallback(*ResourceUnitResult) {}
func (t *transactionAttempt) ReportResourceUnits(units *ResourceUnitResult) {
if units == nil {
return
}
t.recordResourceUnit(units)
}
func (t *transactionAttempt) ReportResourceUnitsError(err error) {
if err == nil {
return
}
var kerr *KeyValueError
if errors.As(err, &kerr) {
if kerr.Internal.ResourceUnits != nil {
t.recordResourceUnit(kerr.Internal.ResourceUnits)
}
}
}
gocbcore-10.2.3/transaction_result.go 0000664 0000000 0000000 00000004064 14417540156 0017645 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
// TransactionAttempt represents a singular attempt at executing a transaction. A
// transaction may require multiple attempts before being successful.
type TransactionAttempt struct {
State TransactionAttemptState
ID string
AtrID []byte
AtrBucketName string
AtrScopeName string
AtrCollectionName string
// UnstagingComplete indicates whether the transaction was succesfully
// unstaged, or if a later cleanup job will be responsible.
UnstagingComplete bool
// Expired indicates whether this attempt expired during execution.
Expired bool
// PreExpiryAutoRollback indicates whether an auto-rollback occured
// before the transaction was expired.
PreExpiryAutoRollback bool
}
// TransactionResult represents the result of a transaction which was executed.
type TransactionResult struct {
// TransactionID represents the UUID assigned to this transaction
TransactionID string
// Attempts records all attempts that were performed when executing
// this transaction.
Attempts []TransactionAttempt
// UnstagingComplete indicates whether the transaction was succesfully
// unstaged, or if a later cleanup job will be responsible.
UnstagingComplete bool
}
// TransactionResourceUnitResult describes the number of resource units used by a transaction attempt.
// Internal: This should never be used and is not supported.
type TransactionResourceUnitResult struct {
NumOps uint32
ReadUnits uint32
WriteUnits uint32
}
gocbcore-10.2.3/transactionattempt.go 0000664 0000000 0000000 00000016565 14417540156 0017657 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"fmt"
"sync/atomic"
"time"
)
type transactionAttempt struct {
// immutable state
expiryTime time.Time
txnStartTime time.Time
keyValueTimeout time.Duration
durabilityLevel TransactionDurabilityLevel
transactionID string
id string
hooks TransactionHooks
enableNonFatalGets bool
enableParallelUnstaging bool
enableExplicitATRs bool
enableMutationCaching bool
atrLocation TransactionATRLocation
bucketAgentProvider TransactionsBucketAgentProviderFn
// mutable state
state TransactionAttemptState
stateBits uint32
stagedMutations []*transactionStagedMutation
atrAgent *Agent
atrOboUser string
atrScopeName string
atrCollectionName string
atrKey []byte
lock asyncMutex
opsWg asyncWaitGroup
hasCleanupRequest bool
addCleanupRequest addCleanupRequest
addLostCleanupLocation addLostCleanupLocation
logger *internalTransactionLogWrapper
recordResourceUnit resourceUnitCallback
}
func (t *transactionAttempt) State() TransactionAttempt {
state := TransactionAttempt{}
t.lock.LockSync()
stateBits := atomic.LoadUint32(&t.stateBits)
state.State = t.state
state.ID = t.id
if stateBits&transactionStateBitHasExpired != 0 {
state.Expired = true
} else {
state.Expired = false
}
if stateBits&transactionStateBitPreExpiryAutoRollback != 0 {
state.PreExpiryAutoRollback = true
} else {
state.PreExpiryAutoRollback = false
}
if t.atrAgent != nil {
state.AtrBucketName = t.atrAgent.BucketName()
state.AtrScopeName = t.atrScopeName
state.AtrCollectionName = t.atrCollectionName
state.AtrID = t.atrKey
} else {
state.AtrBucketName = ""
state.AtrScopeName = ""
state.AtrCollectionName = ""
state.AtrID = []byte{}
}
if t.state == TransactionAttemptStateCompleted {
state.UnstagingComplete = true
} else {
state.UnstagingComplete = false
}
t.lock.UnlockSync()
return state
}
func (t *transactionAttempt) HasExpired() bool {
return t.isExpiryOvertimeAtomic()
}
func (t *transactionAttempt) CanCommit() bool {
stateBits := atomic.LoadUint32(&t.stateBits)
return (stateBits & transactionStateBitShouldNotCommit) == 0
}
func (t *transactionAttempt) ShouldRollback() bool {
stateBits := atomic.LoadUint32(&t.stateBits)
return (stateBits & transactionStateBitShouldNotRollback) == 0
}
func (t *transactionAttempt) ShouldRetry() bool {
stateBits := atomic.LoadUint32(&t.stateBits)
return (stateBits&transactionStateBitShouldNotRetry) == 0 && !t.isExpiryOvertimeAtomic()
}
func (t *transactionAttempt) FinalErrorToRaise() TransactionErrorReason {
stateBits := atomic.LoadUint32(&t.stateBits)
return TransactionErrorReason((stateBits & transactionStateBitsMaskFinalError) >> transactionStateBitsPositionFinalError)
}
func (t *transactionAttempt) UpdateState(opts TransactionUpdateStateOptions) {
t.logger.logInfof(t.id, "Updating state to %s", opts)
stateBits := uint32(0)
if opts.ShouldNotCommit {
stateBits |= transactionStateBitShouldNotCommit
}
if opts.ShouldNotRollback {
stateBits |= transactionStateBitShouldNotRollback
}
if opts.ShouldNotRetry {
stateBits |= transactionStateBitShouldNotRetry
}
if opts.Reason == TransactionErrorReasonTransactionExpired {
stateBits |= transactionStateBitHasExpired
}
t.applyStateBits(stateBits, uint32(opts.Reason))
t.lock.LockSync()
if opts.State > 0 {
t.state = opts.State
}
t.lock.UnlockSync()
}
func (t *transactionAttempt) GetATRLocation() TransactionATRLocation {
t.lock.LockSync()
if t.atrAgent != nil {
location := TransactionATRLocation{
Agent: t.atrAgent,
ScopeName: t.atrScopeName,
CollectionName: t.atrCollectionName,
}
t.lock.UnlockSync()
return location
}
t.lock.UnlockSync()
return t.atrLocation
}
func (t *transactionAttempt) SetATRLocation(location TransactionATRLocation) error {
t.logger.logInfof(t.id, "Setting ATR location to %s", newLoggableATRKey(location.Agent.BucketName(), location.ScopeName, location.CollectionName, nil))
t.lock.LockSync()
if t.atrAgent != nil {
t.lock.UnlockSync()
return errors.New("atr location cannot be set after mutations have occurred")
}
if t.atrLocation.Agent != nil {
t.lock.UnlockSync()
return errors.New("atr location can only be set once")
}
t.atrLocation = location
t.lock.UnlockSync()
return nil
}
func (t *transactionAttempt) GetMutations() []TransactionStagedMutation {
mutations := make([]TransactionStagedMutation, len(t.stagedMutations))
t.lock.LockSync()
for mutationIdx, mutation := range t.stagedMutations {
mutations[mutationIdx] = TransactionStagedMutation{
OpType: mutation.OpType,
BucketName: mutation.Agent.BucketName(),
ScopeName: mutation.ScopeName,
CollectionName: mutation.CollectionName,
Key: mutation.Key,
Cas: mutation.Cas,
Staged: mutation.Staged,
}
}
t.lock.UnlockSync()
return mutations
}
func (t *transactionAttempt) TimeRemaining() time.Duration {
curTime := time.Now()
timeLeft := time.Duration(0)
if curTime.Before(t.expiryTime) {
timeLeft = t.expiryTime.Sub(curTime)
}
return timeLeft
}
func (t *transactionAttempt) Serialize(cb func([]byte, error)) error {
var res jsonSerializedAttempt
t.waitForOpsAndLock(func(unlock func()) {
if err := t.checkCanCommitLocked(); err != nil {
unlock()
cb(nil, err)
return
}
res.ID.Transaction = t.transactionID
res.ID.Attempt = t.id
if t.atrAgent != nil {
res.ATR.Bucket = t.atrAgent.BucketName()
res.ATR.Scope = t.atrScopeName
res.ATR.Collection = t.atrCollectionName
res.ATR.ID = string(t.atrKey)
} else if t.atrLocation.Agent != nil {
res.ATR.Bucket = t.atrLocation.Agent.BucketName()
res.ATR.Scope = t.atrLocation.ScopeName
res.ATR.Collection = t.atrLocation.CollectionName
res.ATR.ID = ""
}
res.Config.KeyValueTimeoutMs = int(t.keyValueTimeout / time.Millisecond)
res.Config.DurabilityLevel = transactionDurabilityLevelToString(t.durabilityLevel)
res.Config.NumAtrs = 1024
res.State.TimeLeftMs = int(t.TimeRemaining().Milliseconds())
for _, mutation := range t.stagedMutations {
var mutationData jsonSerializedMutation
mutationData.Bucket = mutation.Agent.BucketName()
mutationData.Scope = mutation.ScopeName
mutationData.Collection = mutation.CollectionName
mutationData.ID = string(mutation.Key)
mutationData.Cas = fmt.Sprintf("%d", mutation.Cas)
mutationData.Type = transactionStagedMutationTypeToString(mutation.OpType)
res.Mutations = append(res.Mutations, mutationData)
}
if len(res.Mutations) == 0 {
res.Mutations = []jsonSerializedMutation{}
}
unlock()
resBytes, err := json.Marshal(res)
if err != nil {
cb(nil, err)
return
}
cb(resBytes, nil)
})
return nil
}
gocbcore-10.2.3/transactionattempt_atrs.go 0000664 0000000 0000000 00000067773 14417540156 0020717 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func (t *transactionAttempt) selectAtrLocked(
firstAgent *Agent,
firstOboUser string,
firstScopeName string,
firstCollectionName string,
firstKey []byte,
cb func(*TransactionOperationFailedError),
) {
atrID := int(cbcVbMap(firstKey, 1024))
atrKey := []byte(transactionAtrIDList[atrID])
t.hooks.RandomATRIDForVbucket(func(s string, err error) {
if err != nil {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyHookError(err),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
if s != "" {
atrKey = []byte(s)
}
atrAgent := firstAgent
atrOboUser := firstOboUser
atrScopeName := "_default"
atrCollectionName := "_default"
if t.atrLocation.Agent != nil {
atrAgent = t.atrLocation.Agent
atrOboUser = t.atrLocation.OboUser
atrScopeName = t.atrLocation.ScopeName
atrCollectionName = t.atrLocation.CollectionName
} else {
if t.enableExplicitATRs {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(errors.New("atrs must be explicitly defined")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
}
t.atrAgent = atrAgent
t.atrOboUser = atrOboUser
t.atrScopeName = atrScopeName
t.atrCollectionName = atrCollectionName
t.atrKey = atrKey
cb(nil)
})
}
func (t *transactionAttempt) setATRPendingLocked(
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
switch cerr.Class {
case TransactionErrorClassFailAmbiguous:
time.AfterFunc(3*time.Millisecond, func() {
t.setATRPendingLocked(cb)
})
return
case TransactionErrorClassFailPathAlreadyExists:
cb(nil)
return
case TransactionErrorClassFailExpiry:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionExpired,
}))
case TransactionErrorClassFailOutOfSpace:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr.Wrap(ErrAtrFull),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailTransient:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailHard:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
default:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
}
t.checkExpiredAtomic(hookATRPending, []byte{}, false, func(cerr *classifiedError) {
if cerr != nil {
ecCb(cerr)
return
}
t.hooks.BeforeATRPending(func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
var marshalErr error
atrFieldOp := func(fieldName string, data interface{}, flags memd.SubdocFlag) SubDocOp {
b, err := json.Marshal(data)
if err != nil {
marshalErr = err
return SubDocOp{}
}
return SubDocOp{
Op: memd.SubDocOpDictAdd,
Flags: memd.SubdocFlagMkDirP | flags,
Path: "attempts." + t.id + "." + fieldName,
Value: b,
}
}
atrOps := []SubDocOp{
atrFieldOp("tst", "${Mutation.CAS}", memd.SubdocFlagXattrPath|memd.SubdocFlagExpandMacros),
atrFieldOp("tid", t.transactionID, memd.SubdocFlagXattrPath),
atrFieldOp("st", jsonAtrStatePending, memd.SubdocFlagXattrPath),
atrFieldOp("exp", time.Until(t.expiryTime)/time.Millisecond, memd.SubdocFlagXattrPath),
{
Op: memd.SubDocOpSetDoc,
Flags: memd.SubdocFlagNone,
Path: "",
Value: []byte{0},
},
atrFieldOp("d", transactionsDurabilityLevelToShorthand(t.durabilityLevel), memd.SubdocFlagXattrPath),
}
if marshalErr != nil {
ecCb(classifyError(marshalErr))
return
}
t.logger.logInfof(t.id, "Setting ATR %s pending", newLoggableATRKey(
t.atrAgent.BucketName(),
t.atrScopeName,
t.atrCollectionName,
t.atrKey,
))
_, err = t.atrAgent.MutateIn(MutateInOptions{
ScopeName: t.atrScopeName,
CollectionName: t.atrCollectionName,
Key: t.atrKey,
Ops: atrOps,
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
Deadline: deadline,
Flags: memd.SubdocDocFlagMkDoc,
User: t.atrOboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
for _, op := range result.Ops {
if op.Err != nil {
ecCb(classifyError(op.Err))
return
}
}
t.hooks.AfterATRPending(func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
t.addLostCleanupLocation(t.atrAgent.BucketName(), t.atrScopeName, t.atrCollectionName)
ecCb(nil)
})
})
if err != nil {
ecCb(classifyError(err))
return
}
})
})
}
func (t *transactionAttempt) fetchATRCommitConflictLocked(
cb func(jsonAtrState, *TransactionOperationFailedError),
) {
ecCb := func(st jsonAtrState, cerr *classifiedError) {
if cerr == nil {
cb(st, nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
switch cerr.Class {
case TransactionErrorClassFailTransient:
fallthrough
case TransactionErrorClassFailOther:
time.AfterFunc(3*time.Millisecond, func() {
t.fetchATRCommitConflictLocked(cb)
})
return
case TransactionErrorClassFailDocNotFound:
cb(jsonAtrStateUnknown, t.operationFailed(operationFailedDef{
Cerr: cerr.Wrap(ErrAtrNotFound),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionCommitAmbiguous,
}))
case TransactionErrorClassFailPathNotFound:
cb(jsonAtrStateUnknown, t.operationFailed(operationFailedDef{
Cerr: cerr.Wrap(ErrAtrEntryNotFound),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionCommitAmbiguous,
}))
case TransactionErrorClassFailExpiry:
cb(jsonAtrStateUnknown, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionCommitAmbiguous,
}))
case TransactionErrorClassFailHard:
cb(jsonAtrStateUnknown, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionCommitAmbiguous,
}))
default:
cb(jsonAtrStateUnknown, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionCommitAmbiguous,
}))
return
}
}
t.checkExpiredAtomic(hookATRCommitAmbiguityResolution, []byte{}, false, func(cerr *classifiedError) {
if cerr != nil {
ecCb(jsonAtrStateUnknown, cerr)
return
}
t.hooks.BeforeATRCommitAmbiguityResolution(func(err error) {
if err != nil {
ecCb(jsonAtrStateUnknown, classifyHookError(err))
return
}
var deadline time.Time
if t.keyValueTimeout > 0 {
deadline = time.Now().Add(t.keyValueTimeout)
}
_, err = t.atrAgent.LookupIn(LookupInOptions{
ScopeName: t.atrScopeName,
CollectionName: t.atrCollectionName,
Key: t.atrKey,
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "attempts." + t.id + ".st",
Flags: memd.SubdocFlagXattrPath,
},
},
Deadline: deadline,
Flags: memd.SubdocDocFlagNone,
User: t.atrOboUser,
}, func(result *LookupInResult, err error) {
if err != nil {
ecCb(jsonAtrStateUnknown, classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
if result.Ops[0].Err != nil {
ecCb(jsonAtrStateUnknown, classifyError(err))
return
}
var st jsonAtrState
if err := json.Unmarshal(result.Ops[0].Value, &st); err != nil {
ecCb(jsonAtrStateUnknown, classifyError(err))
return
}
ecCb(st, nil)
})
if err != nil {
ecCb(jsonAtrStateUnknown, classifyError(err))
return
}
})
})
}
func (t *transactionAttempt) resolveATRCommitConflictLocked(
cb func(*TransactionOperationFailedError),
) {
t.fetchATRCommitConflictLocked(func(st jsonAtrState, err *TransactionOperationFailedError) {
if err != nil {
cb(err)
return
}
switch st {
case jsonAtrStatePending:
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "transaction still pending even with p set during commit")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
case jsonAtrStateCommitted:
cb(nil)
case jsonAtrStateCompleted:
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "transaction already completed during commit")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
case jsonAtrStateAborted:
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "transaction already aborted during commit")),
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case jsonAtrStateRolledBack:
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "transaction already rolled back during commit")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
default:
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, fmt.Sprintf("illegal transaction state during commit: %s", st))),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
})
}
func (t *transactionAttempt) setATRCommittedLocked(
ambiguityResolution bool,
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
errorReason := TransactionErrorReasonTransactionFailed
if ambiguityResolution {
errorReason = TransactionErrorReasonTransactionCommitAmbiguous
}
switch cerr.Class {
case TransactionErrorClassFailAmbiguous:
time.AfterFunc(3*time.Millisecond, func() {
ambiguityResolution = true
t.setATRCommittedLocked(ambiguityResolution, cb)
})
return
case TransactionErrorClassFailTransient:
if ambiguityResolution {
time.AfterFunc(3*time.Millisecond, func() {
t.setATRCommittedLocked(ambiguityResolution, cb)
})
return
}
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: errorReason,
}))
case TransactionErrorClassFailPathAlreadyExists:
t.resolveATRCommitConflictLocked(cb)
return
case TransactionErrorClassFailDocNotFound:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr.Wrap(ErrAtrNotFound),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: errorReason,
}))
case TransactionErrorClassFailPathNotFound:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr.Wrap(ErrAtrEntryNotFound),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: errorReason,
}))
case TransactionErrorClassFailOutOfSpace:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr.Wrap(ErrAtrFull),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: errorReason,
}))
case TransactionErrorClassFailExpiry:
if errorReason == TransactionErrorReasonTransactionFailed {
errorReason = TransactionErrorReasonTransactionExpired
}
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: errorReason,
}))
case TransactionErrorClassFailHard:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: errorReason,
}))
default:
if ambiguityResolution {
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: errorReason,
}))
return
}
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: errorReason,
}))
}
}
atrAgent := t.atrAgent
atrOboUser := t.atrOboUser
atrScopeName := t.atrScopeName
atrKey := t.atrKey
atrCollectionName := t.atrCollectionName
insMutations := []jsonAtrMutation{}
repMutations := []jsonAtrMutation{}
remMutations := []jsonAtrMutation{}
for _, mutation := range t.stagedMutations {
jsonMutation := jsonAtrMutation{
BucketName: mutation.Agent.BucketName(),
ScopeName: mutation.ScopeName,
CollectionName: mutation.CollectionName,
DocID: string(mutation.Key),
}
if mutation.OpType == TransactionStagedMutationInsert {
insMutations = append(insMutations, jsonMutation)
} else if mutation.OpType == TransactionStagedMutationReplace {
repMutations = append(repMutations, jsonMutation)
} else if mutation.OpType == TransactionStagedMutationRemove {
remMutations = append(remMutations, jsonMutation)
} else {
ecCb(classifyError(wrapError(ErrIllegalState, "unexpected staged mutation type")))
return
}
}
t.checkExpiredAtomic(hookATRCommit, []byte{}, false, func(cerr *classifiedError) {
if cerr != nil {
ecCb(cerr)
return
}
t.hooks.BeforeATRCommit(func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
var marshalErr error
atrFieldOp := func(fieldName string, data interface{}, flags memd.SubdocFlag, op memd.SubDocOpType) SubDocOp {
bytes, err := json.Marshal(data)
if err != nil {
marshalErr = err
}
return SubDocOp{
Op: op,
Flags: flags,
Path: "attempts." + t.id + "." + fieldName,
Value: bytes,
}
}
atrOps := []SubDocOp{
atrFieldOp("st", jsonAtrStateCommitted, memd.SubdocFlagXattrPath, memd.SubDocOpDictSet),
atrFieldOp("tsc", "${Mutation.CAS}", memd.SubdocFlagXattrPath|memd.SubdocFlagExpandMacros, memd.SubDocOpDictSet),
atrFieldOp("p", 0, memd.SubdocFlagXattrPath, memd.SubDocOpDictAdd),
atrFieldOp("ins", insMutations, memd.SubdocFlagXattrPath, memd.SubDocOpDictSet),
atrFieldOp("rep", repMutations, memd.SubdocFlagXattrPath, memd.SubDocOpDictSet),
atrFieldOp("rem", remMutations, memd.SubdocFlagXattrPath, memd.SubDocOpDictSet),
}
if marshalErr != nil {
ecCb(classifyError(marshalErr))
return
}
_, err = atrAgent.MutateIn(MutateInOptions{
ScopeName: atrScopeName,
CollectionName: atrCollectionName,
Key: atrKey,
Ops: atrOps,
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
Flags: memd.SubdocDocFlagNone,
Deadline: deadline,
User: atrOboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
for _, op := range result.Ops {
if op.Err != nil {
ecCb(classifyError(op.Err))
return
}
}
t.hooks.AfterATRCommit(func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
ecCb(nil)
})
})
if err != nil {
ecCb(classifyError(err))
return
}
})
})
}
func (t *transactionAttempt) setATRCompletedLocked(
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
if t.isExpiryOvertimeAtomic() {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "completed atr removal failed during overtime")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
return
}
switch cerr.Class {
case TransactionErrorClassFailDocNotFound:
fallthrough
case TransactionErrorClassFailPathNotFound:
// This is technically a full success, but FIT expects unstagingCompleted=false...
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
case TransactionErrorClassFailExpiry:
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "completed atr removal operation expired")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
case TransactionErrorClassFailHard:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
default:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
}
}
atrAgent := t.atrAgent
atrOboUser := t.atrOboUser
atrScopeName := t.atrScopeName
atrKey := t.atrKey
atrCollectionName := t.atrCollectionName
t.checkExpiredAtomic(hookATRComplete, []byte{}, true, func(cerr *classifiedError) {
if cerr != nil {
ecCb(cerr)
return
}
t.hooks.BeforeATRComplete(func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
atrOps := []SubDocOp{
{
Op: memd.SubDocOpDelete,
Flags: memd.SubdocFlagXattrPath,
Path: "attempts." + t.id,
},
}
_, err = atrAgent.MutateIn(MutateInOptions{
ScopeName: atrScopeName,
CollectionName: atrCollectionName,
Key: atrKey,
Ops: atrOps,
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
Deadline: deadline,
Flags: memd.SubdocDocFlagNone,
User: atrOboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
for _, op := range result.Ops {
if op.Err != nil {
ecCb(classifyError(op.Err))
return
}
}
t.hooks.AfterATRComplete(func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
ecCb(nil)
})
})
if err != nil {
ecCb(classifyError(err))
return
}
})
})
}
func (t *transactionAttempt) setATRAbortedLocked(
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
if t.isExpiryOvertimeAtomic() {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "atr abort failed during overtime")),
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
return
}
switch cerr.Class {
case TransactionErrorClassFailExpiry:
t.setExpiryOvertimeAtomic()
time.AfterFunc(3*time.Millisecond, func() {
t.setATRAbortedLocked(cb)
})
case TransactionErrorClassFailDocNotFound:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr.Wrap(ErrAtrNotFound),
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
case TransactionErrorClassFailPathNotFound:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr.Wrap(ErrAtrEntryNotFound),
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
case TransactionErrorClassFailOutOfSpace:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr.Wrap(ErrAtrFull),
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
case TransactionErrorClassFailHard:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
default:
time.AfterFunc(3*time.Millisecond, func() {
t.setATRAbortedLocked(cb)
})
}
}
atrAgent := t.atrAgent
atrOboUser := t.atrOboUser
atrScopeName := t.atrScopeName
atrKey := t.atrKey
atrCollectionName := t.atrCollectionName
insMutations := []jsonAtrMutation{}
repMutations := []jsonAtrMutation{}
remMutations := []jsonAtrMutation{}
for _, mutation := range t.stagedMutations {
jsonMutation := jsonAtrMutation{
BucketName: mutation.Agent.BucketName(),
ScopeName: mutation.ScopeName,
CollectionName: mutation.CollectionName,
DocID: string(mutation.Key),
}
if mutation.OpType == TransactionStagedMutationInsert {
insMutations = append(insMutations, jsonMutation)
} else if mutation.OpType == TransactionStagedMutationReplace {
repMutations = append(repMutations, jsonMutation)
} else if mutation.OpType == TransactionStagedMutationRemove {
remMutations = append(remMutations, jsonMutation)
} else {
ecCb(classifyError(wrapError(ErrIllegalState, "unexpected staged mutation type")))
return
}
}
t.checkExpiredAtomic(hookATRAbort, []byte{}, true, func(cerr *classifiedError) {
if cerr != nil {
ecCb(cerr)
return
}
t.hooks.BeforeATRAborted(func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
var marshalErr error
atrFieldOp := func(fieldName string, data interface{}, flags memd.SubdocFlag) SubDocOp {
bytes, err := json.Marshal(data)
if err != nil {
marshalErr = err
}
return SubDocOp{
Op: memd.SubDocOpDictSet,
Flags: flags,
Path: "attempts." + t.id + "." + fieldName,
Value: bytes,
}
}
atrOps := []SubDocOp{
atrFieldOp("st", jsonAtrStateAborted, memd.SubdocFlagXattrPath),
atrFieldOp("tsrs", "${Mutation.CAS}", memd.SubdocFlagXattrPath|memd.SubdocFlagExpandMacros),
atrFieldOp("ins", insMutations, memd.SubdocFlagXattrPath),
atrFieldOp("rep", repMutations, memd.SubdocFlagXattrPath),
atrFieldOp("rem", remMutations, memd.SubdocFlagXattrPath),
}
if marshalErr != nil {
ecCb(classifyError(marshalErr))
return
}
_, err = atrAgent.MutateIn(MutateInOptions{
ScopeName: atrScopeName,
CollectionName: atrCollectionName,
Key: atrKey,
Ops: atrOps,
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
Flags: memd.SubdocDocFlagNone,
Deadline: deadline,
User: atrOboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
for _, op := range result.Ops {
if op.Err != nil {
ecCb(classifyError(op.Err))
return
}
}
t.hooks.AfterATRAborted(func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
ecCb(nil)
})
})
if err != nil {
ecCb(classifyError(err))
return
}
})
})
}
func (t *transactionAttempt) setATRRolledBackLocked(
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
if t.isExpiryOvertimeAtomic() {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "rolled back atr removal failed during overtime")),
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
return
}
switch cerr.Class {
case TransactionErrorClassFailDocNotFound:
fallthrough
case TransactionErrorClassFailPathNotFound:
cb(nil)
return
case TransactionErrorClassFailExpiry:
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "rolled back atr removal operation expired")),
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
case TransactionErrorClassFailHard:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
default:
time.AfterFunc(3*time.Millisecond, func() {
t.setATRRolledBackLocked(cb)
})
}
}
atrAgent := t.atrAgent
atrOboUser := t.atrOboUser
atrScopeName := t.atrScopeName
atrKey := t.atrKey
atrCollectionName := t.atrCollectionName
t.checkExpiredAtomic(hookATRRollback, []byte{}, true, func(cerr *classifiedError) {
if cerr != nil {
ecCb(cerr)
return
}
t.hooks.BeforeATRRolledBack(func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
atrOps := []SubDocOp{
{
Op: memd.SubDocOpDelete,
Flags: memd.SubdocFlagXattrPath,
Path: "attempts." + t.id,
},
}
_, err = atrAgent.MutateIn(MutateInOptions{
ScopeName: atrScopeName,
CollectionName: atrCollectionName,
Key: atrKey,
Ops: atrOps,
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
Deadline: deadline,
Flags: memd.SubdocDocFlagNone,
User: atrOboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
for _, op := range result.Ops {
if op.Err != nil {
ecCb(classifyError(op.Err))
return
}
}
t.hooks.AfterATRRolledBack(func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
ecCb(nil)
})
})
if err != nil {
ecCb(classifyError(err))
return
}
})
})
}
gocbcore-10.2.3/transactionattempt_commit.go 0000664 0000000 0000000 00000046143 14417540156 0021222 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func (t *transactionAttempt) Commit(cb TransactionCommitCallback) error {
t.logger.logInfof(t.id, "Performing commit")
return t.commit(func(err *TransactionOperationFailedError) {
if err != nil {
t.logger.logInfof(t.id, "Commit failed")
if t.ShouldRollback() {
if !t.isExpiryOvertimeAtomic() {
t.applyStateBits(transactionStateBitPreExpiryAutoRollback, 0)
}
err := t.rollback(func(rerr *TransactionOperationFailedError) {
if rerr != nil {
t.logger.logInfof(t.id, "Rollback failed")
logDebugf("implicit rollback after commit failure errored: %s", rerr)
}
t.ensureCleanUpRequest()
cb(err)
})
if err != nil {
t.logger.logInfof(t.id, "Rollback failed to schedule")
logDebugf("failed to schedule rollback after commit failure errored: %s", err)
t.ensureCleanUpRequest()
cb(err)
}
return
}
t.ensureCleanUpRequest()
cb(err)
return
}
t.applyStateBits(transactionStateBitShouldNotRetry|transactionStateBitShouldNotRollback, 0)
t.ensureCleanUpRequest()
cb(nil)
})
}
func (t *transactionAttempt) commit(
cb func(err *TransactionOperationFailedError),
) error {
t.waitForOpsAndLock(func(unlock func()) {
unlockAndCb := func(err *TransactionOperationFailedError) {
unlock()
cb(err)
}
err := t.checkCanCommitLocked()
if err != nil {
unlockAndCb(err)
return
}
t.applyStateBits(transactionStateBitShouldNotCommit, 0)
if t.state == TransactionAttemptStateNothingWritten {
unlockAndCb(nil)
return
}
t.checkExpiredAtomic(hookCommit, []byte{}, false, func(cerr *classifiedError) {
if cerr != nil {
unlockAndCb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionExpired,
}))
return
}
t.state = TransactionAttemptStateCommitting
t.setATRCommittedLocked(false, func(err *TransactionOperationFailedError) {
if err != nil {
if err.shouldRaise == TransactionErrorReasonTransactionFailedPostCommit {
t.state = TransactionAttemptStateCommitted
} else if err.shouldRaise != TransactionErrorReasonTransactionCommitAmbiguous {
t.state = TransactionAttemptStatePending
}
unlockAndCb(err)
return
}
t.state = TransactionAttemptStateCommitted
go func() {
commitStagedMutation := func(
mutation *transactionStagedMutation,
unstageCb func(*TransactionOperationFailedError),
) {
t.fetchBeforeUnstage(mutation, func(err *TransactionOperationFailedError) {
if err != nil {
unstageCb(err)
return
}
switch mutation.OpType {
case TransactionStagedMutationInsert:
t.commitStagedInsert(*mutation, false, unstageCb)
case TransactionStagedMutationReplace:
t.commitStagedReplace(*mutation, false, false, unstageCb)
case TransactionStagedMutationRemove:
t.commitStagedRemove(*mutation, false, unstageCb)
default:
unstageCb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "unexpected staged mutation type")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
}
})
}
var mutErrs []*TransactionOperationFailedError
if !t.enableParallelUnstaging {
for _, mutation := range t.stagedMutations {
waitCh := make(chan struct{}, 1)
commitStagedMutation(mutation, func(err *TransactionOperationFailedError) {
if err != nil {
mutErrs = append(mutErrs, err)
waitCh <- struct{}{}
return
}
waitCh <- struct{}{}
})
<-waitCh
if len(mutErrs) > 0 {
break
}
}
} else {
type mutResult struct {
Err *TransactionOperationFailedError
}
numMutations := len(t.stagedMutations)
waitCh := make(chan mutResult, numMutations)
// Unlike the RFC we do insert and replace separately. We have a bug in gocbcore where subdocs
// will raise doc exists rather than a cas mismatch so we need to do these ops separately to tell
// how to handle that error.
for _, mutation := range t.stagedMutations {
commitStagedMutation(mutation, func(err *TransactionOperationFailedError) {
waitCh <- mutResult{
Err: err,
}
})
}
for i := 0; i < numMutations; i++ {
res := <-waitCh
if res.Err != nil {
mutErrs = append(mutErrs, res.Err)
continue
}
}
}
err = mergeOperationFailedErrors(mutErrs)
if err != nil {
unlockAndCb(err)
return
}
t.setATRCompletedLocked(func(err *TransactionOperationFailedError) {
if err != nil {
if err.errorClass != TransactionErrorClassFailHard {
unlockAndCb(nil)
return
}
unlockAndCb(err)
return
}
t.state = TransactionAttemptStateCompleted
unlockAndCb(nil)
})
}()
})
})
})
return nil
}
func (t *transactionAttempt) fetchBeforeUnstage(
mutation *transactionStagedMutation,
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
if t.isExpiryOvertimeAtomic() {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "fetching staged data failed during overtime")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
return
}
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
}
if mutation.OpType != TransactionStagedMutationInsert && mutation.OpType != TransactionStagedMutationReplace {
ecCb(nil)
return
}
if mutation.Staged != nil {
ecCb(nil)
return
}
t.checkExpiredAtomic(hookCommitDoc, mutation.Key, false, func(cerr *classifiedError) {
if cerr != nil {
t.setExpiryOvertimeAtomic()
}
var flags memd.SubdocDocFlag
if mutation.OpType == TransactionStagedMutationInsert {
flags = memd.SubdocDocFlagAccessDeleted
}
var deadline time.Time
if t.keyValueTimeout > 0 {
deadline = time.Now().Add(t.keyValueTimeout)
}
_, err := mutation.Agent.LookupIn(LookupInOptions{
ScopeName: mutation.ScopeName,
CollectionName: mutation.CollectionName,
Key: mutation.Key,
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
},
Deadline: deadline,
Flags: flags,
User: mutation.OboUser,
}, func(result *LookupInResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
if result.Ops[0].Err != nil {
ecCb(classifyError(result.Ops[0].Err))
return
}
var jsonTxn jsonTxnXattr
err = json.Unmarshal(result.Ops[0].Value, &jsonTxn)
if err != nil {
ecCb(classifyError(err))
return
}
if jsonTxn.ID.Attempt != t.id {
ecCb(classifyError(ErrOther))
return
}
mutation.Cas = result.Cas
mutation.Staged = jsonTxn.Operation.Staged
ecCb(nil)
})
if err != nil {
ecCb(classifyError(err))
return
}
})
}
func (t *transactionAttempt) commitStagedReplace(
mutation transactionStagedMutation,
forceWrite bool,
ambiguityResolution bool,
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
if t.isExpiryOvertimeAtomic() {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "committing a replace failed during overtime")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
return
}
switch cerr.Class {
case TransactionErrorClassFailAmbiguous:
time.AfterFunc(3*time.Millisecond, func() {
ambiguityResolution = true
t.commitStagedReplace(mutation, forceWrite, ambiguityResolution, cb)
})
case TransactionErrorClassFailDocAlreadyExists:
cerr.Class = TransactionErrorClassFailCasMismatch
fallthrough
case TransactionErrorClassFailCasMismatch:
if !ambiguityResolution {
time.AfterFunc(3*time.Millisecond, func() {
forceWrite = true
t.commitStagedReplace(mutation, forceWrite, ambiguityResolution, cb)
})
return
}
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
case TransactionErrorClassFailDocNotFound:
t.commitStagedInsert(mutation, ambiguityResolution, cb)
return
case TransactionErrorClassFailExpiry:
t.setExpiryOvertimeAtomic()
time.AfterFunc(3*time.Millisecond, func() {
t.commitStagedReplace(mutation, forceWrite, ambiguityResolution, cb)
})
case TransactionErrorClassFailHard:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
default:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
}
}
t.checkExpiredAtomic(hookCommitDoc, mutation.Key, false, func(cerr *classifiedError) {
if cerr != nil {
t.setExpiryOvertimeAtomic()
}
t.hooks.BeforeDocCommitted(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
cas := mutation.Cas
if forceWrite {
cas = 0
}
if mutation.Staged == nil {
ecCb(classifyError(
wrapError(ErrIllegalState, "staged content is missing")))
}
_, err = mutation.Agent.MutateIn(MutateInOptions{
ScopeName: mutation.ScopeName,
CollectionName: mutation.CollectionName,
Key: mutation.Key,
Cas: cas,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
Value: []byte{110, 117, 108, 108}, // null
},
{
Op: memd.SubDocOpDelete,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
{
Op: memd.SubDocOpSetDoc,
Path: "",
Value: mutation.Staged,
},
},
Deadline: deadline,
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
User: mutation.OboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
for _, op := range result.Ops {
if op.Err != nil {
ecCb(classifyError(op.Err))
return
}
}
t.hooks.AfterDocCommittedBeforeSavingCAS(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
t.hooks.AfterDocCommitted(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
ecCb(nil)
})
})
})
if err != nil {
ecCb(classifyError(err))
return
}
})
})
}
func (t *transactionAttempt) commitStagedInsert(
mutation transactionStagedMutation,
ambiguityResolution bool,
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
if t.isExpiryOvertimeAtomic() {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "committing an insert failed during overtime")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
return
}
switch cerr.Class {
case TransactionErrorClassFailAmbiguous:
time.AfterFunc(3*time.Millisecond, func() {
ambiguityResolution = true
t.commitStagedInsert(mutation, ambiguityResolution, cb)
})
case TransactionErrorClassFailDocAlreadyExists:
cerr.Class = TransactionErrorClassFailCasMismatch
fallthrough
case TransactionErrorClassFailCasMismatch:
if !ambiguityResolution {
time.AfterFunc(3*time.Millisecond, func() {
t.commitStagedReplace(mutation, true, ambiguityResolution, cb)
})
return
}
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
case TransactionErrorClassFailExpiry:
t.setExpiryOvertimeAtomic()
time.AfterFunc(3*time.Millisecond, func() {
t.commitStagedInsert(mutation, ambiguityResolution, cb)
})
case TransactionErrorClassFailHard:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
default:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
}
}
t.checkExpiredAtomic(hookCommitDoc, mutation.Key, false, func(cerr *classifiedError) {
if cerr != nil {
t.setExpiryOvertimeAtomic()
}
t.hooks.BeforeDocCommitted(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
if mutation.Staged == nil {
ecCb(classifyError(
wrapError(ErrIllegalState, "staged content is missing")))
}
_, err = mutation.Agent.Add(AddOptions{
ScopeName: mutation.ScopeName,
CollectionName: mutation.CollectionName,
Key: mutation.Key,
Value: mutation.Staged,
Deadline: deadline,
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
User: mutation.OboUser,
}, func(result *StoreResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
t.hooks.AfterDocCommittedBeforeSavingCAS(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
t.hooks.AfterDocCommitted(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
ecCb(nil)
})
})
})
if err != nil {
ecCb(classifyError(err))
return
}
})
})
}
func (t *transactionAttempt) commitStagedRemove(
mutation transactionStagedMutation,
ambiguityResolution bool,
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
if t.isExpiryOvertimeAtomic() {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "committing a remove failed during overtime")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
return
}
switch cerr.Class {
case TransactionErrorClassFailAmbiguous:
time.AfterFunc(3*time.Millisecond, func() {
ambiguityResolution = true
t.commitStagedRemove(mutation, ambiguityResolution, cb)
})
return
case TransactionErrorClassFailDocNotFound:
// Not finding the document during ambiguity resolution likely indicates
// that it simply successfully performed the operation already. However, the mutation
// token of that won't be available, so we need to just error it anyways :(
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
case TransactionErrorClassFailExpiry:
t.setExpiryOvertimeAtomic()
time.AfterFunc(3*time.Millisecond, func() {
t.commitStagedRemove(mutation, ambiguityResolution, cb)
})
case TransactionErrorClassFailHard:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
default:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailedPostCommit,
}))
}
}
t.checkExpiredAtomic(hookCommitDoc, mutation.Key, false, func(cerr *classifiedError) {
if cerr != nil {
t.setExpiryOvertimeAtomic()
}
t.hooks.BeforeDocRemoved(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
_, err = mutation.Agent.Delete(DeleteOptions{
ScopeName: mutation.ScopeName,
CollectionName: mutation.CollectionName,
Key: mutation.Key,
Cas: 0,
Deadline: deadline,
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
User: mutation.OboUser,
}, func(result *DeleteResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
t.hooks.AfterDocRemovedPreRetry(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
t.hooks.AfterDocRemovedPostRetry(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
ecCb(nil)
})
})
})
if err != nil {
ecCb(classifyError(err))
return
}
})
})
}
gocbcore-10.2.3/transactionattempt_erroring.go 0000664 0000000 0000000 00000012427 14417540156 0021557 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"errors"
"sync/atomic"
)
func mergeOperationFailedErrors(errs []*TransactionOperationFailedError) *TransactionOperationFailedError {
if len(errs) == 0 {
return nil
}
if len(errs) == 1 {
return errs[0]
}
shouldNotRetry := false
shouldNotRollback := false
aggCauses := aggregateError{}
shouldRaise := TransactionErrorReasonTransactionFailed
for errIdx := 0; errIdx < len(errs); errIdx++ {
tErr := errs[errIdx]
aggCauses = append(aggCauses, tErr)
if tErr.shouldNotRetry {
shouldNotRetry = true
}
if tErr.shouldNotRollback {
shouldNotRollback = true
}
if tErr.shouldRaise > shouldRaise {
shouldRaise = tErr.shouldRaise
}
}
return &TransactionOperationFailedError{
shouldNotRetry: shouldNotRetry,
shouldNotRollback: shouldNotRollback,
errorCause: aggCauses,
shouldRaise: shouldRaise,
errorClass: TransactionErrorClassFailOther,
}
}
type operationFailedDef struct {
Cerr *classifiedError
ShouldNotRetry bool
ShouldNotRollback bool
CanStillCommit bool
Reason TransactionErrorReason
}
func (t *transactionAttempt) applyStateBits(stateBits uint32, errorBits uint32) {
// This is a bit dirty, but its maximum going to do one retry per bit.
for {
oldStateBits := atomic.LoadUint32(&t.stateBits)
newStateBits := oldStateBits | stateBits
if errorBits > ((oldStateBits & transactionStateBitsMaskFinalError) >> transactionStateBitsPositionFinalError) {
newStateBits = (newStateBits & transactionStateBitsMaskBits) | (errorBits << transactionStateBitsPositionFinalError)
}
t.logger.logInfof(t.id, "Applying state bits: %08b, error bits: %08b, old: %08b, new: %08b",
stateBits, errorBits, oldStateBits, newStateBits)
if atomic.CompareAndSwapUint32(&t.stateBits, oldStateBits, newStateBits) {
break
}
}
}
func (t *transactionAttempt) operationFailed(def operationFailedDef) *TransactionOperationFailedError {
t.logger.logInfof(t.id, "Operation failed: can still commit: %t, should not rollback: %t, should not retry: %t, "+
"reason: %s", def.CanStillCommit, def.ShouldNotRollback, def.ShouldNotRetry, def.Reason)
err := &TransactionOperationFailedError{
shouldNotRetry: def.ShouldNotRetry,
shouldNotRollback: def.ShouldNotRollback,
errorCause: def.Cerr.Source,
errorClass: def.Cerr.Class,
shouldRaise: def.Reason,
}
stateBits := uint32(0)
if !def.CanStillCommit {
stateBits |= transactionStateBitShouldNotCommit
}
if def.ShouldNotRollback {
stateBits |= transactionStateBitShouldNotRollback
}
if def.ShouldNotRetry {
stateBits |= transactionStateBitShouldNotRetry
}
if def.Reason == TransactionErrorReasonTransactionExpired {
stateBits |= transactionStateBitHasExpired
}
t.applyStateBits(stateBits, uint32(def.Reason))
return err
}
func classifyHookError(err error) *classifiedError {
// We currently have to classify the errors that are returned from the hooks, but
// we should really just directly return the classifications and make the source
// some special internal source showing it came from a hook...
return classifyError(err)
}
func classifyError(err error) *classifiedError {
ec := TransactionErrorClassFailOther
if errors.Is(err, ErrDocAlreadyInTransaction) || errors.Is(err, ErrWriteWriteConflict) {
ec = TransactionErrorClassFailWriteWriteConflict
} else if errors.Is(err, ErrHard) {
ec = TransactionErrorClassFailHard
} else if errors.Is(err, ErrAttemptExpired) {
ec = TransactionErrorClassFailExpiry
} else if errors.Is(err, ErrTransient) {
ec = TransactionErrorClassFailTransient
} else if errors.Is(err, ErrDocumentNotFound) {
ec = TransactionErrorClassFailDocNotFound
} else if errors.Is(err, ErrAmbiguous) {
ec = TransactionErrorClassFailAmbiguous
} else if errors.Is(err, ErrCasMismatch) {
ec = TransactionErrorClassFailCasMismatch
} else if errors.Is(err, ErrDocumentNotFound) {
ec = TransactionErrorClassFailDocNotFound
} else if errors.Is(err, ErrDocumentExists) {
ec = TransactionErrorClassFailDocAlreadyExists
} else if errors.Is(err, ErrPathExists) {
ec = TransactionErrorClassFailPathAlreadyExists
} else if errors.Is(err, ErrPathNotFound) {
ec = TransactionErrorClassFailPathNotFound
} else if errors.Is(err, ErrCasMismatch) {
ec = TransactionErrorClassFailCasMismatch
} else if errors.Is(err, ErrUnambiguousTimeout) {
ec = TransactionErrorClassFailTransient
} else if errors.Is(err, ErrDurabilityAmbiguous) ||
errors.Is(err, ErrAmbiguousTimeout) ||
errors.Is(err, ErrRequestCanceled) {
ec = TransactionErrorClassFailAmbiguous
} else if errors.Is(err, ErrMemdTooBig) || errors.Is(err, ErrValueTooLarge) {
ec = TransactionErrorClassFailOutOfSpace
}
return &classifiedError{
Source: err,
Class: ec,
}
}
gocbcore-10.2.3/transactionattempt_get.go 0000664 0000000 0000000 00000031414 14417540156 0020504 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func (t *transactionAttempt) Get(opts TransactionGetOptions, cb TransactionGetCallback) error {
return t.get(opts, func(res *TransactionGetResult, err error) {
if err != nil {
t.logger.logInfof(t.id, "Get failed %s", err)
if !t.ShouldRollback() {
t.ensureCleanUpRequest()
}
cb(nil, err)
return
}
cb(res, nil)
})
}
func (t *transactionAttempt) get(
opts TransactionGetOptions,
cb func(*TransactionGetResult, error),
) error {
forceNonFatal := t.enableNonFatalGets
t.logger.logInfof(t.id, "Performing get for %s non fatal enabled: %t", newLoggableDocKey(
opts.Agent.BucketName(),
opts.ScopeName,
opts.CollectionName,
opts.Key,
), forceNonFatal)
t.beginOpAndLock(func(unlock func(), endOp func()) {
endAndCb := func(result *TransactionGetResult, err error) {
endOp()
cb(result, err)
}
err := t.checkCanPerformOpLocked()
if err != nil {
unlock()
endAndCb(nil, err)
return
}
unlock()
t.checkExpiredAtomic(hookGet, opts.Key, false, func(cerr *classifiedError) {
if cerr != nil {
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionExpired,
}))
return
}
t.mavRead(opts.Agent, opts.OboUser, opts.ScopeName, opts.CollectionName, opts.Key, opts.NoRYOW,
"", forceNonFatal, func(result *TransactionGetResult, err error) {
if err != nil {
endAndCb(nil, err)
return
}
t.hooks.AfterGetComplete(opts.Key, func(err error) {
if err != nil {
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyHookError(err),
CanStillCommit: forceNonFatal,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
endAndCb(result, nil)
})
})
})
})
return nil
}
func (t *transactionAttempt) mavRead(
agent *Agent,
oboUser string,
scopeName string,
collectionName string,
key []byte,
disableRYOW bool,
resolvingATREntry string,
forceNonFatal bool,
cb func(*TransactionGetResult, error),
) {
t.fetchDocWithMeta(
agent,
oboUser,
scopeName,
collectionName,
key,
forceNonFatal,
func(doc *transactionGetDoc, err error) {
if err != nil {
cb(nil, err)
return
}
if disableRYOW {
if doc.TxnMeta != nil && doc.TxnMeta.ID.Attempt == t.id {
t.logger.logInfof(t.id, "Disable RYOW set and tnx meta is not nil, resetting meta to nil")
// This is going to be a RYOW, we can just clear the TxnMeta which
// will cause us to fall into the block below.
doc.TxnMeta = nil
}
}
// Doc not involved in another transaction.
if doc.TxnMeta == nil {
if doc.Deleted {
cb(nil, wrapError(ErrDocumentNotFound, "doc was a tombstone"))
return
}
t.logger.logInfof(t.id, "Txn meta is nil, returning result")
cb(&TransactionGetResult{
agent: agent,
oboUser: oboUser,
scopeName: scopeName,
collectionName: collectionName,
key: key,
Value: doc.Body,
Cas: doc.Cas,
Meta: nil,
}, nil)
return
}
if doc.TxnMeta.ID.Attempt == t.id {
switch doc.TxnMeta.Operation.Type {
case jsonMutationInsert:
t.logger.logInfof(t.id, "Doc already in txn as insert, using staged value")
cb(&TransactionGetResult{
agent: agent,
oboUser: oboUser,
scopeName: scopeName,
collectionName: collectionName,
key: key,
Value: doc.TxnMeta.Operation.Staged,
Cas: doc.Cas,
}, nil)
case jsonMutationReplace:
t.logger.logInfof(t.id, "Doc already in txn as replace, using staged value")
cb(&TransactionGetResult{
agent: agent,
oboUser: oboUser,
scopeName: scopeName,
collectionName: collectionName,
key: key,
Value: doc.TxnMeta.Operation.Staged,
Cas: doc.Cas,
}, nil)
case jsonMutationRemove:
cb(nil, wrapError(ErrDocumentNotFound, "doc was a staged remove"))
default:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "unexpected staged mutation type")),
CanStillCommit: forceNonFatal,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
return
}
if doc.TxnMeta.ID.Attempt == resolvingATREntry {
if doc.Deleted {
cb(nil, wrapError(ErrDocumentNotFound, "doc was a staged tombstone during resolution"))
return
}
t.logger.logInfof(t.id, "Completed ATR resolution")
cb(&TransactionGetResult{
agent: agent,
oboUser: oboUser,
scopeName: scopeName,
collectionName: collectionName,
key: key,
Value: doc.Body,
Cas: doc.Cas,
}, nil)
return
}
docFc := jsonForwardCompatToForwardCompat(doc.TxnMeta.ForwardCompat)
docMeta := &TransactionMutableItemMeta{
TransactionID: doc.TxnMeta.ID.Transaction,
AttemptID: doc.TxnMeta.ID.Attempt,
ATR: TransactionMutableItemMetaATR{
BucketName: doc.TxnMeta.ATR.BucketName,
ScopeName: doc.TxnMeta.ATR.ScopeName,
CollectionName: doc.TxnMeta.ATR.CollectionName,
DocID: doc.TxnMeta.ATR.DocID,
},
ForwardCompat: docFc,
}
t.checkForwardCompatability(
key,
agent.BucketName(),
scopeName,
collectionName,
forwardCompatStageGets,
docFc,
forceNonFatal,
func(err *TransactionOperationFailedError) {
if err != nil {
cb(nil, err)
return
}
t.getTxnState(
agent.BucketName(),
scopeName,
collectionName,
key,
doc.TxnMeta.ATR.BucketName,
doc.TxnMeta.ATR.ScopeName,
doc.TxnMeta.ATR.CollectionName,
doc.TxnMeta.ATR.DocID,
doc.TxnMeta.ID.Attempt,
forceNonFatal,
func(attempt *jsonAtrAttempt, expiry time.Time, err *TransactionOperationFailedError) {
if err != nil {
cb(nil, err)
return
}
if attempt == nil {
t.logger.logInfof(t.id, "ATR entry missing, rerunning mav read")
// The ATR entry is missing, it's likely that we just raced the other transaction
// cleaning up it's documents and then cleaning itself up. Lets run ATR resolution.
t.mavRead(agent, oboUser, scopeName, collectionName, key, disableRYOW, doc.TxnMeta.ID.Attempt, forceNonFatal, cb)
return
}
atmptFc := jsonForwardCompatToForwardCompat(attempt.ForwardCompat)
t.checkForwardCompatability(
key,
agent.BucketName(),
scopeName,
collectionName,
forwardCompatStageGetsReadingATR, atmptFc, forceNonFatal, func(err *TransactionOperationFailedError) {
if err != nil {
cb(nil, err)
return
}
state := jsonAtrState(attempt.State)
if state == jsonAtrStateCommitted || state == jsonAtrStateCompleted {
switch doc.TxnMeta.Operation.Type {
case jsonMutationInsert:
t.logger.logInfof(t.id, "Doc already in txn as insert, using staged value")
cb(&TransactionGetResult{
agent: agent,
oboUser: oboUser,
scopeName: scopeName,
collectionName: collectionName,
key: key,
Value: doc.TxnMeta.Operation.Staged,
Cas: doc.Cas,
Meta: docMeta,
}, nil)
case jsonMutationReplace:
t.logger.logInfof(t.id, "Doc already in txn as replace, using staged value")
cb(&TransactionGetResult{
agent: agent,
oboUser: oboUser,
scopeName: scopeName,
collectionName: collectionName,
key: key,
Value: doc.TxnMeta.Operation.Staged,
Cas: doc.Cas,
Meta: docMeta,
}, nil)
case jsonMutationRemove:
cb(nil, wrapError(ErrDocumentNotFound, "doc was a staged remove"))
default:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "unexpected staged mutation type")),
ShouldNotRetry: false,
ShouldNotRollback: false,
}))
}
return
}
if doc.Deleted {
cb(nil, wrapError(ErrDocumentNotFound, "doc was a tombstone"))
return
}
cb(&TransactionGetResult{
agent: agent,
oboUser: oboUser,
scopeName: scopeName,
collectionName: collectionName,
key: key,
Value: doc.Body,
Cas: doc.Cas,
Meta: docMeta,
}, nil)
})
})
})
})
}
func (t *transactionAttempt) fetchDocWithMeta(
agent *Agent,
oboUser string,
scopeName string,
collectionName string,
key []byte,
forceNonFatal bool,
cb func(*transactionGetDoc, error),
) {
ecCb := func(doc *transactionGetDoc, cerr *classifiedError) {
if cerr == nil {
cb(doc, nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
switch cerr.Class {
case TransactionErrorClassFailDocNotFound:
cb(nil, wrapError(ErrDocumentNotFound, "doc was not found"))
case TransactionErrorClassFailTransient:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
CanStillCommit: forceNonFatal,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailHard:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
CanStillCommit: forceNonFatal,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
default:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
CanStillCommit: forceNonFatal,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
}
t.hooks.BeforeDocGet(key, func(err error) {
if err != nil {
ecCb(nil, classifyHookError(err))
return
}
var deadline time.Time
if t.keyValueTimeout > 0 {
deadline = time.Now().Add(t.keyValueTimeout)
}
_, err = agent.LookupIn(LookupInOptions{
ScopeName: scopeName,
CollectionName: collectionName,
Key: key,
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "$document",
Flags: memd.SubdocFlagXattrPath,
},
{
Op: memd.SubDocOpGet,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
{
Op: memd.SubDocOpGetDoc,
Path: "",
Flags: 0,
},
},
Deadline: deadline,
Flags: memd.SubdocDocFlagAccessDeleted,
User: oboUser,
}, func(result *LookupInResult, err error) {
if err != nil {
ecCb(nil, classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
if result.Ops[0].Err != nil {
ecCb(nil, classifyError(result.Ops[0].Err))
return
}
var meta *transactionDocMeta
if err := json.Unmarshal(result.Ops[0].Value, &meta); err != nil {
ecCb(nil, classifyError(err))
return
}
var txnMeta *jsonTxnXattr
if result.Ops[1].Err == nil {
// Doc is currently in a txn.
var txnMetaVal jsonTxnXattr
if err := json.Unmarshal(result.Ops[1].Value, &txnMetaVal); err != nil {
ecCb(nil, classifyError(err))
return
}
txnMeta = &txnMetaVal
}
var docBody []byte
if result.Ops[2].Err == nil {
docBody = result.Ops[2].Value
}
ecCb(&transactionGetDoc{
Body: docBody,
TxnMeta: txnMeta,
DocMeta: meta,
Cas: result.Cas,
Deleted: result.Internal.IsDeleted,
}, nil)
})
if err != nil {
ecCb(nil, classifyError(err))
}
})
}
gocbcore-10.2.3/transactionattempt_helpers.go 0000664 0000000 0000000 00000052300 14417540156 0021364 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"bytes"
"encoding/json"
"fmt"
"sync/atomic"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func transactionHasExpired(expiryTime time.Time) bool {
return time.Now().After(expiryTime)
}
func (t *transactionAttempt) beginOpAndLock(cb func(unlock func(), endOp func())) {
t.lock.Lock(func(unlock func()) {
t.opsWg.Add(1)
cb(unlock, func() {
t.opsWg.Done()
})
})
}
func (t *transactionAttempt) waitForOpsAndLock(cb func(unlock func())) {
var tryWaitAndLock func()
tryWaitAndLock = func() {
t.opsWg.Wait(func() {
t.lock.Lock(func(unlock func()) {
if !t.opsWg.IsEmpty() {
unlock()
tryWaitAndLock()
return
}
cb(unlock)
})
})
}
tryWaitAndLock()
}
func (t *transactionAttempt) checkCanPerformOpLocked() *TransactionOperationFailedError {
switch t.state {
case TransactionAttemptStateNothingWritten:
fallthrough
case TransactionAttemptStatePending:
// Good to continue
case TransactionAttemptStateCommitting:
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "transaction is ambiguously committed")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
})
case TransactionAttemptStateCommitted:
fallthrough
case TransactionAttemptStateCompleted:
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "transaction already committed")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
})
case TransactionAttemptStateAborted:
fallthrough
case TransactionAttemptStateRolledBack:
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "transaction already aborted")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
})
default:
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, fmt.Sprintf("invalid transaction state: %v", t.state))),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
})
}
stateBits := atomic.LoadUint32(&t.stateBits)
if (stateBits & transactionStateBitShouldNotCommit) != 0 {
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrPreviousOperationFailed, "previous operation prevents further operations")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
})
}
return nil
}
func (t *transactionAttempt) checkCanCommitRollbackLocked() *TransactionOperationFailedError {
switch t.state {
case TransactionAttemptStateNothingWritten:
fallthrough
case TransactionAttemptStatePending:
// Good to continue
case TransactionAttemptStateCommitting:
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "transaction is ambiguously committed")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
})
case TransactionAttemptStateCommitted:
fallthrough
case TransactionAttemptStateCompleted:
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "transaction already committed")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
})
case TransactionAttemptStateAborted:
fallthrough
case TransactionAttemptStateRolledBack:
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "transaction already aborted")),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
})
default:
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, fmt.Sprintf("invalid transaction state: %v", t.state))),
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
})
}
return nil
}
func (t *transactionAttempt) checkCanCommitLocked() *TransactionOperationFailedError {
err := t.checkCanCommitRollbackLocked()
if err != nil {
return err
}
stateBits := atomic.LoadUint32(&t.stateBits)
if (stateBits & transactionStateBitShouldNotCommit) != 0 {
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrPreviousOperationFailed, "previous operation prevents commit")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
})
}
return nil
}
func (t *transactionAttempt) checkCanRollbackLocked() *TransactionOperationFailedError {
err := t.checkCanCommitRollbackLocked()
if err != nil {
return err
}
stateBits := atomic.LoadUint32(&t.stateBits)
if (stateBits & transactionStateBitShouldNotRollback) != 0 {
return t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrPreviousOperationFailed, "previous operation prevents rollback")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
})
}
return nil
}
func (t *transactionAttempt) setExpiryOvertimeAtomic() {
t.logger.logInfof(t.id, "Entering expiry overtime")
t.applyStateBits(transactionStateBitHasExpired, 0)
}
func (t *transactionAttempt) isExpiryOvertimeAtomic() bool {
stateBits := atomic.LoadUint32(&t.stateBits)
return (stateBits & transactionStateBitHasExpired) != 0
}
func (t *transactionAttempt) checkExpiredAtomic(stage string, id []byte, proceedInOvertime bool, cb func(*classifiedError)) {
if proceedInOvertime && t.isExpiryOvertimeAtomic() {
cb(nil)
return
}
t.hooks.HasExpiredClientSideHook(stage, id, func(expired bool, err error) {
if err != nil {
cb(classifyError(wrapError(err, "HasExpired hook returned an unexpected error")))
return
}
if expired {
cb(classifyError(wrapError(ErrAttemptExpired, "a hook has marked this attempt expired")))
return
} else if transactionHasExpired(t.expiryTime) {
cb(classifyError(wrapError(ErrAttemptExpired, "the expiry for the attempt was reached")))
return
}
cb(nil)
})
}
func (t *transactionAttempt) confirmATRPending(
firstAgent *Agent,
firstOboUser string,
firstScopeName string,
firstCollectionName string,
firstKey []byte,
cb func(*TransactionOperationFailedError),
) {
t.lock.Lock(func(unlock func()) {
unlockAndCb := func(err *TransactionOperationFailedError) {
unlock()
cb(err)
}
if t.state != TransactionAttemptStateNothingWritten {
unlockAndCb(nil)
return
}
t.selectAtrLocked(
firstAgent,
firstOboUser,
firstScopeName,
firstCollectionName,
firstKey,
func(err *TransactionOperationFailedError) {
if err != nil {
unlockAndCb(err)
return
}
t.setATRPendingLocked(func(err *TransactionOperationFailedError) {
if err != nil {
unlockAndCb(err)
return
}
t.state = TransactionAttemptStatePending
unlockAndCb(nil)
})
})
})
}
func (t *transactionAttempt) getStagedMutationLocked(
bucketName, scopeName, collectionName string, key []byte,
) (int, *transactionStagedMutation) {
for i, mutation := range t.stagedMutations {
if mutation.Agent.BucketName() == bucketName &&
mutation.ScopeName == scopeName &&
mutation.CollectionName == collectionName &&
bytes.Equal(mutation.Key, key) {
return i, mutation
}
}
return -1, nil
}
func (t *transactionAttempt) removeStagedMutation(
bucketName, scopeName, collectionName string, key []byte,
cb func(),
) {
t.lock.Lock(func(unlock func()) {
mutIdx, _ := t.getStagedMutationLocked(bucketName, scopeName, collectionName, key)
if mutIdx >= 0 {
// Not finding the item should be basically impossible, but we wrap it just in case...
t.stagedMutations = append(t.stagedMutations[:mutIdx], t.stagedMutations[mutIdx+1:]...)
}
unlock()
cb()
})
}
func (t *transactionAttempt) recordStagedMutation(
stagedInfo *transactionStagedMutation,
cb func(),
) {
if !t.enableMutationCaching {
stagedInfo.Staged = nil
}
t.lock.Lock(func(unlock func()) {
mutIdx, _ := t.getStagedMutationLocked(
stagedInfo.Agent.BucketName(),
stagedInfo.ScopeName,
stagedInfo.CollectionName,
stagedInfo.Key)
if mutIdx >= 0 {
t.stagedMutations[mutIdx] = stagedInfo
} else {
t.stagedMutations = append(t.stagedMutations, stagedInfo)
}
unlock()
cb()
})
}
func (t *transactionAttempt) checkForwardCompatability(
key []byte,
bucket, scope, collection string,
stage forwardCompatStage,
fc map[string][]TransactionForwardCompatibilityEntry,
forceNonFatal bool,
cb func(*TransactionOperationFailedError),
) {
t.logger.logInfof(t.id, "Checking forward compatibility")
isCompat, shouldRetry, retryWait, err := checkForwardCompatability(stage, fc)
if err != nil {
t.logger.logInfof(t.id, "Forward compatability error")
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(err),
CanStillCommit: forceNonFatal,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
if !isCompat {
if shouldRetry {
cbRetryError := func() {
t.logger.logInfof(t.id, "Forward compatability failed - incompatible, should retry")
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(forwardCompatError{
BucketName: bucket,
ScopeName: scope,
CollectionName: collection,
DocumentKey: key,
}),
CanStillCommit: forceNonFatal,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
if retryWait > 0 {
time.AfterFunc(retryWait, cbRetryError)
} else {
cbRetryError()
}
return
}
t.logger.logInfof(t.id, "Forward compatability failed - incompatible")
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(forwardCompatError{
BucketName: bucket,
ScopeName: scope,
CollectionName: collection,
DocumentKey: key,
}),
CanStillCommit: forceNonFatal,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
cb(nil)
}
func (t *transactionAttempt) getTxnState(
srcBucketName string,
srcScopeName string,
srcCollectionName string,
srcDocID []byte,
atrBucketName string,
atrScopeName string,
atrCollectionName string,
atrDocID string,
attemptID string,
forceNonFatal bool,
cb func(*jsonAtrAttempt, time.Time, *TransactionOperationFailedError),
) {
ecCb := func(res *jsonAtrAttempt, txnExp time.Time, cerr *classifiedError) {
if cerr == nil {
cb(res, txnExp, nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
switch cerr.Class {
case TransactionErrorClassFailPathNotFound:
t.logger.logInfof(t.id, "Attempt entry not found")
// If the path is not found, we just return as if there was no
// entry data available for that atr entry.
cb(nil, time.Time{}, nil)
case TransactionErrorClassFailDocNotFound:
t.logger.logInfof(t.id, "ATR doc not found")
// If the ATR is not found, we just return as if there was no
// entry data available for that atr entry.
cb(nil, time.Time{}, nil)
default:
cb(nil, time.Time{}, t.operationFailed(operationFailedDef{
Cerr: classifyError(&writeWriteConflictError{
Source: cerr.Source,
BucketName: srcBucketName,
ScopeName: srcScopeName,
CollectionName: srcCollectionName,
DocumentKey: srcDocID,
}),
CanStillCommit: forceNonFatal,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
}
t.logger.logInfof(t.id, "Getting txn state")
atrAgent, atrOboUser, err := t.bucketAgentProvider(atrBucketName)
if err != nil {
t.logger.logInfof(t.id, "Failed to get atr agent")
ecCb(nil, time.Time{}, classifyError(err))
return
}
t.hooks.BeforeCheckATREntryForBlockingDoc([]byte(atrDocID), func(err error) {
if err != nil {
ecCb(nil, time.Time{}, classifyHookError(err))
return
}
var deadline time.Time
if t.keyValueTimeout > 0 {
deadline = time.Now().Add(t.keyValueTimeout)
}
_, err = atrAgent.LookupIn(LookupInOptions{
ScopeName: atrScopeName,
CollectionName: atrCollectionName,
Key: []byte(atrDocID),
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "attempts." + attemptID,
Flags: memd.SubdocFlagXattrPath,
},
{
Op: memd.SubDocOpGet,
Path: hlcMacro,
Flags: memd.SubdocFlagXattrPath,
},
},
Deadline: deadline,
User: atrOboUser,
}, func(result *LookupInResult, err error) {
if err != nil {
ecCb(nil, time.Time{}, classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
for _, op := range result.Ops {
if op.Err != nil {
ecCb(nil, time.Time{}, classifyError(op.Err))
return
}
}
var txnAttempt *jsonAtrAttempt
if err := json.Unmarshal(result.Ops[0].Value, &txnAttempt); err != nil {
ecCb(nil, time.Time{}, classifyError(err))
return
}
var hlc *jsonHLC
if err := json.Unmarshal(result.Ops[1].Value, &hlc); err != nil {
ecCb(nil, time.Time{}, classifyError(err))
return
}
nowSecs, err := parseHLCToSeconds(*hlc)
if err != nil {
ecCb(nil, time.Time{}, classifyError(err))
return
}
txnStartMs, err := parseCASToMilliseconds(txnAttempt.PendingCAS)
if err != nil {
ecCb(nil, time.Time{}, classifyError(err))
return
}
nowTime := time.Duration(nowSecs) * time.Second
txnStartTime := time.Duration(txnStartMs) * time.Millisecond
txnExpiryTime := time.Duration(txnAttempt.ExpiryTime) * time.Millisecond
txnElapsedTime := nowTime - txnStartTime
txnExpiry := time.Now().Add(txnExpiryTime - txnElapsedTime)
ecCb(txnAttempt, txnExpiry, nil)
})
if err != nil {
ecCb(nil, time.Time{}, classifyError(err))
return
}
})
}
func (t *transactionAttempt) writeWriteConflictPoll(
stage forwardCompatStage,
agent *Agent,
oboUser string,
scopeName string,
collectionName string,
key []byte,
cas Cas,
meta *TransactionMutableItemMeta,
existingMutation *transactionStagedMutation,
cb func(*TransactionOperationFailedError),
) {
if meta == nil {
t.logger.logInfof(t.id, "Meta is nil, no write-write conflict")
// There is no write-write conflict.
cb(nil)
return
}
if meta.TransactionID == t.transactionID {
if meta.AttemptID == t.id {
if existingMutation != nil {
if cas != existingMutation.Cas {
// There was an existing mutation but it doesn't match the expected
// CAS. We throw a CAS mismatch to early detect this.
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrCasMismatch, "cas mismatch occured against local staged mutation")),
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
cb(nil)
return
}
// This means that we are trying to overwrite a previous write this specific
// attempt has performed without actually having found the existing mutation,
// this is never going to work correctly.
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "attempted to overwrite local staged mutation but couldn't find it")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
t.logger.logInfof(t.id, "Transaction meta matches ours, no write-write conflict")
// The transaction matches our transaction. We can safely overwrite the existing
// data in the txn meta and continue.
cb(nil)
return
}
deadline := time.Now().Add(1 * time.Second)
var onePoll func()
onePoll = func() {
t.logger.logInfof(t.id, "Performing write-write conflict poll")
if !time.Now().Before(deadline) {
t.logger.logInfof(t.id, "Deadline expired during write-write poll")
// If the deadline expired, lets just immediately return.
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(&writeWriteConflictError{
Source: fmt.Errorf(
"deadline expired before WWC was resolved on %s.%s.%s.%s",
meta.ATR.BucketName,
meta.ATR.ScopeName,
meta.ATR.CollectionName,
meta.ATR.DocID),
BucketName: agent.BucketName(),
ScopeName: scopeName,
CollectionName: collectionName,
DocumentKey: key,
}),
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
t.checkForwardCompatability(key, agent.BucketName(), scopeName, collectionName, stage, meta.ForwardCompat, false,
func(err *TransactionOperationFailedError) {
if err != nil {
cb(err)
return
}
t.checkExpiredAtomic(hookWWC, key, false, func(cerr *classifiedError) {
if cerr != nil {
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionExpired,
}))
return
}
t.getTxnState(
agent.BucketName(),
scopeName,
collectionName,
key,
meta.ATR.BucketName,
meta.ATR.ScopeName,
meta.ATR.CollectionName,
meta.ATR.DocID,
meta.AttemptID,
false,
func(attempt *jsonAtrAttempt, expiry time.Time, err *TransactionOperationFailedError) {
if err != nil {
cb(err)
return
}
if attempt == nil {
t.logger.logInfof(t.id, "ATR entry missing, completing write-write conflict poll")
// The ATR entry is missing, which counts as it being completed.
cb(nil)
return
}
state := jsonAtrState(attempt.State)
if state == jsonAtrStateCompleted || state == jsonAtrStateRolledBack {
t.logger.logInfof(t.id, "Attempt state %s, completing write-write conflict poll", state)
// If we have progressed enough to continue, let's do that.
cb(nil)
return
}
time.AfterFunc(200*time.Millisecond, onePoll)
})
})
})
}
onePoll()
}
func (t *transactionAttempt) ensureCleanUpRequest() {
// BUG(TXNG-59): Do not use a synchronous lock for cleanup requests.
// Because of the need to include the state of the transaction within the cleanup
// request, we are not able to do registration until the end of commit/rollback,
// which means that we no longer have the lock on the transaction, and need to
// relock it.
t.lock.LockSync()
if t.state == TransactionAttemptStateCompleted || t.state == TransactionAttemptStateRolledBack {
t.lock.UnlockSync()
t.logger.logInfof(t.id, "Attempt state completed or rolled back, will not add cleanup request")
return
}
if t.hasCleanupRequest {
t.lock.UnlockSync()
t.logger.logInfof(t.id, "Attempt already created cleanup request, will not add cleanup request")
return
}
t.hasCleanupRequest = true
var inserts []TransactionsDocRecord
var replaces []TransactionsDocRecord
var removes []TransactionsDocRecord
for _, staged := range t.stagedMutations {
dr := TransactionsDocRecord{
CollectionName: staged.CollectionName,
ScopeName: staged.ScopeName,
BucketName: staged.Agent.BucketName(),
ID: staged.Key,
}
switch staged.OpType {
case TransactionStagedMutationInsert:
inserts = append(inserts, dr)
case TransactionStagedMutationReplace:
replaces = append(replaces, dr)
case TransactionStagedMutationRemove:
removes = append(removes, dr)
}
}
var bucketName string
if t.atrAgent != nil {
bucketName = t.atrAgent.BucketName()
}
cleanupState := t.state
if cleanupState == TransactionAttemptStateCommitting {
cleanupState = TransactionAttemptStatePending
}
req := &TransactionsCleanupRequest{
AttemptID: t.id,
AtrID: t.atrKey,
AtrCollectionName: t.atrCollectionName,
AtrScopeName: t.atrScopeName,
AtrBucketName: bucketName,
Inserts: inserts,
Replaces: replaces,
Removes: removes,
State: cleanupState,
ForwardCompat: nil, // Let's just be explicit about this, it'll change in the future anyway.
DurabilityLevel: t.durabilityLevel,
Age: time.Since(t.txnStartTime),
}
t.lock.UnlockSync()
t.logger.logInfof(t.id, "Adding cleanup request for atr %s, cleanup state: %s", newLoggableATRKey(
bucketName,
t.atrScopeName,
t.atrCollectionName,
t.atrKey,
), cleanupState)
t.addCleanupRequest(req)
}
gocbcore-10.2.3/transactionattempt_insert.go 0000664 0000000 0000000 00000036776 14417540156 0021251 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func (t *transactionAttempt) Insert(opts TransactionInsertOptions, cb TransactionStoreCallback) error {
return t.insert(opts, func(res *TransactionGetResult, err error) {
if err != nil {
var e *TransactionOperationFailedError
if errors.As(err, &e) {
if e.shouldNotRollback {
t.ensureCleanUpRequest()
}
}
cb(nil, err)
return
}
cb(res, nil)
})
}
func (t *transactionAttempt) insert(
opts TransactionInsertOptions,
cb func(*TransactionGetResult, error),
) error {
t.logger.logInfof(t.id, "Performing insert for %s", newLoggableDocKey(
opts.Agent.BucketName(),
opts.ScopeName,
opts.CollectionName,
opts.Key,
))
t.beginOpAndLock(func(unlock func(), endOp func()) {
endAndCb := func(result *TransactionGetResult, err error) {
endOp()
cb(result, err)
}
err := t.checkCanPerformOpLocked()
if err != nil {
unlock()
endAndCb(nil, err)
return
}
agent := opts.Agent
oboUser := opts.OboUser
scopeName := opts.ScopeName
collectionName := opts.CollectionName
key := opts.Key
value := opts.Value
t.checkExpiredAtomic(hookInsert, key, false, func(cerr *classifiedError) {
if cerr != nil {
unlock()
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionExpired,
}))
return
}
_, existingMutation := t.getStagedMutationLocked(agent.BucketName(), scopeName, collectionName, key)
unlock()
if existingMutation != nil {
switch existingMutation.OpType {
case TransactionStagedMutationRemove:
t.logger.logInfof(t.id, "Staged remove exists on doc, performing replace")
t.stageReplace(
agent, oboUser, scopeName, collectionName, key,
value, existingMutation.Cas,
func(result *TransactionGetResult, err error) {
endAndCb(result, err)
})
return
case TransactionStagedMutationInsert:
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrDocumentExists, "attempted to insert a document previously inserted in this transaction")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
case TransactionStagedMutationReplace:
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrDocumentExists, "attempted to insert a document previously replaced in this transaction")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
default:
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "unexpected staged mutation type")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
}
t.confirmATRPending(agent, oboUser, scopeName, collectionName, key, func(err *TransactionOperationFailedError) {
if err != nil {
endAndCb(nil, err)
return
}
t.stageInsert(
agent, oboUser, scopeName, collectionName, key,
value, 0,
func(result *TransactionGetResult, err error) {
endAndCb(result, err)
})
})
})
})
return nil
}
func (t *transactionAttempt) resolveConflictedInsert(
agent *Agent,
oboUser string,
scopeName string,
collectionName string,
key []byte,
value json.RawMessage,
cb func(*TransactionGetResult, error),
) {
t.getMetaForConflictedInsert(agent, oboUser, scopeName, collectionName, key,
func(isTombstone bool, txnMeta *jsonTxnXattr, cas Cas, err error) {
if err != nil {
cb(nil, err)
return
}
if txnMeta == nil {
// This doc isn't in a transaction
if !isTombstone {
cb(nil, ErrDocumentExists)
return
}
// There wasn't actually a staged mutation there.
t.stageInsert(agent, oboUser, scopeName, collectionName, key, value, cas, cb)
return
}
meta := &TransactionMutableItemMeta{
TransactionID: txnMeta.ID.Transaction,
AttemptID: txnMeta.ID.Attempt,
ATR: TransactionMutableItemMetaATR{
BucketName: txnMeta.ATR.BucketName,
ScopeName: txnMeta.ATR.ScopeName,
CollectionName: txnMeta.ATR.CollectionName,
DocID: txnMeta.ATR.DocID,
},
ForwardCompat: jsonForwardCompatToForwardCompat(txnMeta.ForwardCompat),
}
t.checkForwardCompatability(
key,
agent.BucketName(),
scopeName,
collectionName,
forwardCompatStageWWCInsertingGet, meta.ForwardCompat, false, func(err *TransactionOperationFailedError) {
if err != nil {
cb(nil, err)
return
}
if txnMeta.Operation.Type != jsonMutationInsert {
cb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrDocumentExists, "found staged non-insert mutation")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
// We have guards in place within the write write conflict polling to prevent miss-use when
// an existing mutation must have been discovered before it's safe to overwrite. This logic
// is unneccessary, as is the forwards compatibility check when resolving conflicted inserts
// so we can safely just ignore it.
if meta.TransactionID == t.transactionID && meta.AttemptID == t.id {
t.stageInsert(agent, oboUser, scopeName, collectionName, key, value, cas, cb)
return
}
t.writeWriteConflictPoll(forwardCompatStageWWCInserting, agent, oboUser, scopeName, collectionName, key, cas, meta, nil, func(err *TransactionOperationFailedError) {
if err != nil {
cb(nil, err)
return
}
t.cleanupStagedInsert(agent, oboUser, scopeName, collectionName, key, cas, isTombstone, func(cas Cas, err *TransactionOperationFailedError) {
if err != nil {
cb(nil, err)
return
}
t.stageInsert(agent, oboUser, scopeName, collectionName, key, value, cas, cb)
})
})
})
})
}
func (t *transactionAttempt) stageInsert(
agent *Agent,
oboUser string,
scopeName string,
collectionName string,
key []byte,
value json.RawMessage,
cas Cas,
cb func(*TransactionGetResult, error),
) {
ecCb := func(result *TransactionGetResult, cerr *classifiedError) {
if cerr == nil {
cb(result, nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
switch cerr.Class {
case TransactionErrorClassFailAmbiguous:
time.AfterFunc(3*time.Millisecond, func() {
t.stageInsert(agent, oboUser, scopeName, collectionName, key, value, cas, cb)
})
case TransactionErrorClassFailExpiry:
t.setExpiryOvertimeAtomic()
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionExpired,
}))
case TransactionErrorClassFailTransient:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailHard:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailDocAlreadyExists:
fallthrough
case TransactionErrorClassFailCasMismatch:
t.resolveConflictedInsert(agent, oboUser, scopeName, collectionName, key, value, cb)
default:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
}
t.checkExpiredAtomic(hookInsert, key, false, func(cerr *classifiedError) {
if cerr != nil {
ecCb(nil, cerr)
return
}
t.hooks.BeforeStagedInsert(key, func(err error) {
if err != nil {
ecCb(nil, classifyHookError(err))
return
}
stagedInfo := &transactionStagedMutation{
OpType: TransactionStagedMutationInsert,
Agent: agent,
OboUser: oboUser,
ScopeName: scopeName,
CollectionName: collectionName,
Key: key,
Staged: value,
}
var txnMeta jsonTxnXattr
txnMeta.ID.Transaction = t.transactionID
txnMeta.ID.Attempt = t.id
txnMeta.ATR.CollectionName = t.atrCollectionName
txnMeta.ATR.ScopeName = t.atrScopeName
txnMeta.ATR.BucketName = t.atrAgent.BucketName()
txnMeta.ATR.DocID = string(t.atrKey)
txnMeta.Operation.Type = jsonMutationInsert
txnMeta.Operation.Staged = stagedInfo.Staged
txnMetaBytes, err := json.Marshal(txnMeta)
if err != nil {
ecCb(nil, classifyError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
flags := memd.SubdocDocFlagCreateAsDeleted | memd.SubdocDocFlagAccessDeleted
var txnOp memd.SubDocOpType
if cas == 0 {
flags |= memd.SubdocDocFlagAddDoc
txnOp = memd.SubDocOpDictAdd
} else {
txnOp = memd.SubDocOpDictSet
}
_, err = stagedInfo.Agent.MutateIn(MutateInOptions{
ScopeName: stagedInfo.ScopeName,
CollectionName: stagedInfo.CollectionName,
Key: stagedInfo.Key,
Cas: cas,
Ops: []SubDocOp{
{
Op: txnOp,
Path: "txn",
Flags: memd.SubdocFlagMkDirP | memd.SubdocFlagXattrPath,
Value: txnMetaBytes,
},
{
Op: memd.SubDocOpDictSet,
Path: "txn.op.crc32",
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
Value: crc32cMacro,
},
},
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
Deadline: deadline,
Flags: flags,
User: stagedInfo.OboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(nil, classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
stagedInfo.Cas = result.Cas
t.hooks.AfterStagedInsertComplete(key, func(err error) {
if err != nil {
ecCb(nil, classifyHookError(err))
return
}
t.recordStagedMutation(stagedInfo, func() {
ecCb(&TransactionGetResult{
agent: stagedInfo.Agent,
oboUser: stagedInfo.OboUser,
scopeName: stagedInfo.ScopeName,
collectionName: stagedInfo.CollectionName,
key: stagedInfo.Key,
Value: stagedInfo.Staged,
Cas: stagedInfo.Cas,
Meta: nil,
}, nil)
})
})
})
if err != nil {
ecCb(nil, classifyError(err))
}
})
})
}
func (t *transactionAttempt) getMetaForConflictedInsert(
agent *Agent,
oboUser string,
scopeName string,
collectionName string,
key []byte,
cb func(bool, *jsonTxnXattr, Cas, error),
) {
ecCb := func(isTombstone bool, meta *jsonTxnXattr, cas Cas, cerr *classifiedError) {
if cerr == nil {
cb(isTombstone, meta, cas, nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
switch cerr.Class {
case TransactionErrorClassFailDocNotFound:
fallthrough
case TransactionErrorClassFailPathNotFound:
fallthrough
case TransactionErrorClassFailTransient:
cb(isTombstone, nil, 0, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
default:
cb(isTombstone, nil, 0, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
}
t.hooks.BeforeGetDocInExistsDuringStagedInsert(key, func(err error) {
if err != nil {
ecCb(false, nil, 0, classifyHookError(err))
return
}
var deadline time.Time
if t.keyValueTimeout > 0 {
deadline = time.Now().Add(t.keyValueTimeout)
}
_, err = agent.LookupIn(LookupInOptions{
ScopeName: scopeName,
CollectionName: collectionName,
Key: key,
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
},
Deadline: deadline,
Flags: memd.SubdocDocFlagAccessDeleted,
User: oboUser,
}, func(result *LookupInResult, err error) {
if err != nil {
ecCb(false, nil, 0, classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
var txnMeta *jsonTxnXattr
if result.Ops[0].Err == nil {
var txnMetaVal jsonTxnXattr
if err := json.Unmarshal(result.Ops[0].Value, &txnMetaVal); err != nil {
ecCb(false, nil, 0, classifyError(err))
return
}
txnMeta = &txnMetaVal
}
isTombstone := result.Internal.IsDeleted
ecCb(isTombstone, txnMeta, result.Cas, nil)
})
if err != nil {
ecCb(false, nil, 0, classifyError(err))
return
}
})
}
func (t *transactionAttempt) cleanupStagedInsert(
agent *Agent,
oboUser string,
scopeName string,
collectionName string,
key []byte,
cas Cas,
isTombstone bool,
cb func(Cas, *TransactionOperationFailedError),
) {
ecCb := func(cas Cas, cerr *classifiedError) {
if cerr == nil {
cb(cas, nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
switch cerr.Class {
case TransactionErrorClassFailDocNotFound:
fallthrough
case TransactionErrorClassFailCasMismatch:
fallthrough
case TransactionErrorClassFailTransient:
cb(0, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
default:
cb(0, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
}
if isTombstone {
// This is already a tombstone, so we can just proceed.
ecCb(cas, nil)
return
}
t.hooks.BeforeRemovingDocDuringStagedInsert(key, func(err error) {
if err != nil {
ecCb(0, classifyHookError(err))
return
}
var deadline time.Time
if t.keyValueTimeout > 0 {
deadline = time.Now().Add(t.keyValueTimeout)
}
_, err = agent.Delete(DeleteOptions{
ScopeName: scopeName,
CollectionName: collectionName,
Key: key,
Deadline: deadline,
User: oboUser,
}, func(result *DeleteResult, err error) {
if err != nil {
ecCb(0, classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
ecCb(result.Cas, nil)
})
if err != nil {
ecCb(0, classifyError(err))
return
}
})
}
gocbcore-10.2.3/transactionattempt_remove.go 0000664 0000000 0000000 00000033755 14417540156 0021234 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"github.com/couchbase/gocbcore/v10/memd"
)
func (t *transactionAttempt) Remove(opts TransactionRemoveOptions, cb TransactionStoreCallback) error {
return t.remove(opts, func(res *TransactionGetResult, err *TransactionOperationFailedError) {
if err != nil {
t.logger.logInfof(t.id, "Remove failed")
if err.shouldNotRollback {
t.ensureCleanUpRequest()
}
cb(nil, err)
return
}
cb(res, nil)
})
}
func (t *transactionAttempt) remove(
opts TransactionRemoveOptions,
cb func(*TransactionGetResult, *TransactionOperationFailedError),
) error {
t.logger.logInfof(t.id, "Performing remove for %s", newLoggableDocKey(
opts.Document.agent.BucketName(),
opts.Document.scopeName,
opts.Document.collectionName,
opts.Document.key,
))
t.beginOpAndLock(func(unlock func(), endOp func()) {
endAndCb := func(result *TransactionGetResult, err *TransactionOperationFailedError) {
endOp()
cb(result, err)
}
err := t.checkCanPerformOpLocked()
if err != nil {
unlock()
endAndCb(nil, err)
return
}
agent := opts.Document.agent
oboUser := opts.Document.oboUser
scopeName := opts.Document.scopeName
collectionName := opts.Document.collectionName
key := opts.Document.key
cas := opts.Document.Cas
meta := opts.Document.Meta
t.checkExpiredAtomic(hookRemove, key, false, func(cerr *classifiedError) {
if cerr != nil {
unlock()
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionExpired,
}))
return
}
_, existingMutation := t.getStagedMutationLocked(agent.BucketName(), scopeName, collectionName, key)
unlock()
if existingMutation != nil {
switch existingMutation.OpType {
case TransactionStagedMutationInsert:
t.logger.logInfof(t.id, "Staged insert exists on doc, removing txn metadata")
t.stageRemoveOfInsert(
agent, oboUser, scopeName, collectionName, key,
cas,
func(result *TransactionGetResult, err *TransactionOperationFailedError) {
endAndCb(result, err)
})
return
case TransactionStagedMutationReplace:
t.logger.logInfof(t.id, "Staged replace exists on doc, this is ok")
// We can overwrite other replaces without issue, any conflicts between the mutation
// the user passed to us and the existing mutation is caught by WriteWriteConflict.
case TransactionStagedMutationRemove:
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrDocumentNotFound, "attempted to remove a document previously removed in this transaction")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
default:
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "unexpected staged mutation type")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
}
t.writeWriteConflictPoll(
forwardCompatStageWWCRemoving,
agent, oboUser, scopeName, collectionName, key, cas,
meta,
existingMutation,
func(err *TransactionOperationFailedError) {
if err != nil {
endAndCb(nil, err)
return
}
t.confirmATRPending(agent, oboUser, scopeName, collectionName, key, func(err *TransactionOperationFailedError) {
if err != nil {
endAndCb(nil, err)
return
}
t.stageRemove(
agent, oboUser, scopeName, collectionName, key,
cas,
func(result *TransactionGetResult, err *TransactionOperationFailedError) {
endAndCb(result, err)
})
})
})
})
})
return nil
}
func (t *transactionAttempt) stageRemove(
agent *Agent,
oboUser string,
scopeName string,
collectionName string,
key []byte,
cas Cas,
cb func(*TransactionGetResult, *TransactionOperationFailedError),
) {
ecCb := func(result *TransactionGetResult, cerr *classifiedError) {
if cerr == nil {
cb(result, nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
switch cerr.Class {
case TransactionErrorClassFailExpiry:
t.setExpiryOvertimeAtomic()
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionExpired,
}))
case TransactionErrorClassFailDocNotFound:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrDocumentNotFound, "document not found during staging")),
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailDocAlreadyExists:
cerr.Class = TransactionErrorClassFailCasMismatch
fallthrough
case TransactionErrorClassFailCasMismatch:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailTransient:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailAmbiguous:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailHard:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
default:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
}
t.checkExpiredAtomic(hookRemove, key, false, func(cerr *classifiedError) {
if cerr != nil {
ecCb(nil, cerr)
return
}
t.hooks.BeforeStagedRemove(key, func(err error) {
if err != nil {
ecCb(nil, classifyHookError(err))
return
}
stagedInfo := &transactionStagedMutation{
OpType: TransactionStagedMutationRemove,
Agent: agent,
OboUser: oboUser,
ScopeName: scopeName,
CollectionName: collectionName,
Key: key,
}
var txnMeta jsonTxnXattr
txnMeta.ID.Transaction = t.transactionID
txnMeta.ID.Attempt = t.id
txnMeta.ATR.CollectionName = t.atrCollectionName
txnMeta.ATR.ScopeName = t.atrScopeName
txnMeta.ATR.BucketName = t.atrAgent.BucketName()
txnMeta.ATR.DocID = string(t.atrKey)
txnMeta.Operation.Type = jsonMutationRemove
txnMeta.Restore = &jsonTxnXattrRestore{
OriginalCAS: "",
ExpiryTime: 0,
RevID: "",
}
txnMetaBytes, err := json.Marshal(txnMeta)
if err != nil {
ecCb(nil, classifyError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
flags := memd.SubdocDocFlagAccessDeleted
_, err = stagedInfo.Agent.MutateIn(MutateInOptions{
ScopeName: stagedInfo.ScopeName,
CollectionName: stagedInfo.CollectionName,
Key: stagedInfo.Key,
Cas: cas,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "txn",
Flags: memd.SubdocFlagMkDirP | memd.SubdocFlagXattrPath,
Value: txnMetaBytes,
},
{
Op: memd.SubDocOpDictSet,
Path: "txn.op.crc32",
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
Value: crc32cMacro,
},
{
Op: memd.SubDocOpDictSet,
Path: "txn.restore.CAS",
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
Value: casMacro,
},
{
Op: memd.SubDocOpDictSet,
Path: "txn.restore.exptime",
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
Value: exptimeMacro,
},
{
Op: memd.SubDocOpDictSet,
Path: "txn.restore.revid",
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
Value: revidMacro,
},
},
Flags: flags,
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
Deadline: deadline,
User: stagedInfo.OboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(nil, classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
stagedInfo.Cas = result.Cas
t.hooks.AfterStagedRemoveComplete(key, func(err error) {
if err != nil {
ecCb(nil, classifyHookError(err))
return
}
t.recordStagedMutation(stagedInfo, func() {
ecCb(&TransactionGetResult{
agent: stagedInfo.Agent,
oboUser: stagedInfo.OboUser,
scopeName: stagedInfo.ScopeName,
collectionName: stagedInfo.CollectionName,
key: stagedInfo.Key,
Value: stagedInfo.Staged,
Cas: stagedInfo.Cas,
Meta: nil,
}, nil)
})
})
})
if err != nil {
ecCb(nil, classifyError(err))
return
}
})
})
}
func (t *transactionAttempt) stageRemoveOfInsert(
agent *Agent,
oboUser string,
scopeName string,
collectionName string,
key []byte,
cas Cas,
cb func(*TransactionGetResult, *TransactionOperationFailedError),
) {
ecCb := func(result *TransactionGetResult, cerr *classifiedError) {
if cerr == nil {
cb(result, nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
switch cerr.Class {
case TransactionErrorClassFailExpiry:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionExpired,
}))
case TransactionErrorClassFailDocNotFound:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrDocumentNotFound, "staged document was modified since insert")),
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailDocAlreadyExists:
cerr.Class = TransactionErrorClassFailCasMismatch
fallthrough
case TransactionErrorClassFailCasMismatch:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailTransient:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailAmbiguous:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailHard:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
default:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
}
t.checkExpiredAtomic(hookRemoveStagedInsert, key, false, func(cerr *classifiedError) {
if cerr != nil {
ecCb(nil, cerr)
return
}
t.hooks.BeforeRemoveStagedInsert(key, func(err error) {
if err != nil {
ecCb(nil, classifyHookError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
_, err = agent.MutateIn(MutateInOptions{
ScopeName: scopeName,
CollectionName: collectionName,
Key: key,
Cas: cas,
Flags: memd.SubdocDocFlagAccessDeleted,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDelete,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
},
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
Deadline: deadline,
User: oboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(nil, classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
t.hooks.AfterRemoveStagedInsert(key, func(err error) {
if err != nil {
ecCb(nil, classifyHookError(err))
return
}
t.removeStagedMutation(agent.BucketName(), scopeName, collectionName, key, func() {
cb(&TransactionGetResult{
agent: agent,
oboUser: oboUser,
scopeName: scopeName,
collectionName: collectionName,
key: key,
Cas: result.Cas,
}, nil)
})
})
})
if err != nil {
ecCb(nil, classifyError(err))
return
}
})
})
}
gocbcore-10.2.3/transactionattempt_replace.go 0000664 0000000 0000000 00000024156 14417540156 0021345 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"github.com/couchbase/gocbcore/v10/memd"
)
func (t *transactionAttempt) Replace(opts TransactionReplaceOptions, cb TransactionStoreCallback) error {
return t.replace(opts, func(res *TransactionGetResult, err error) {
if err != nil {
t.logger.logInfof(t.id, "Replace failed")
var e *TransactionOperationFailedError
if errors.As(err, &e) {
if e.shouldNotRollback {
t.ensureCleanUpRequest()
}
}
cb(nil, err)
return
}
cb(res, nil)
})
}
func (t *transactionAttempt) replace(
opts TransactionReplaceOptions,
cb func(*TransactionGetResult, error),
) error {
t.logger.logInfof(t.id, "Performing replace for %s", newLoggableDocKey(
opts.Document.agent.BucketName(),
opts.Document.scopeName,
opts.Document.collectionName,
opts.Document.key,
))
t.beginOpAndLock(func(unlock func(), endOp func()) {
endAndCb := func(result *TransactionGetResult, err error) {
endOp()
cb(result, err)
}
err := t.checkCanPerformOpLocked()
if err != nil {
unlock()
endAndCb(nil, err)
return
}
agent := opts.Document.agent
oboUser := opts.Document.oboUser
scopeName := opts.Document.scopeName
collectionName := opts.Document.collectionName
key := opts.Document.key
value := opts.Value
cas := opts.Document.Cas
meta := opts.Document.Meta
t.checkExpiredAtomic(hookReplace, key, false, func(cerr *classifiedError) {
if cerr != nil {
unlock()
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionExpired,
}))
return
}
_, existingMutation := t.getStagedMutationLocked(agent.BucketName(), scopeName, collectionName, key)
unlock()
if existingMutation != nil {
switch existingMutation.OpType {
case TransactionStagedMutationInsert:
t.logger.logInfof(t.id, "Staged insert exists on doc, performing insert")
t.stageInsert(
agent, oboUser, scopeName, collectionName, key,
value, cas,
func(result *TransactionGetResult, err error) {
endAndCb(result, err)
})
return
case TransactionStagedMutationReplace:
t.logger.logInfof(t.id, "Staged replace exists on doc, this is ok")
// We can overwrite other replaces without issue, any conflicts between the mutation
// the user passed to us and the existing mutation is caught by WriteWriteConflict.
case TransactionStagedMutationRemove:
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrDocumentNotFound, "attempted to replace a document previously removed in this transaction")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
default:
endAndCb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "unexpected staged mutation type")),
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
return
}
}
t.writeWriteConflictPoll(
forwardCompatStageWWCReplacing,
agent, oboUser, scopeName, collectionName, key, cas,
meta,
existingMutation,
func(err *TransactionOperationFailedError) {
if err != nil {
endAndCb(nil, err)
return
}
t.confirmATRPending(agent, oboUser, scopeName, collectionName, key, func(err *TransactionOperationFailedError) {
if err != nil {
endAndCb(nil, err)
return
}
t.stageReplace(
agent, oboUser, scopeName, collectionName, key,
value, cas,
func(result *TransactionGetResult, err error) {
endAndCb(result, err)
})
})
})
})
})
return nil
}
func (t *transactionAttempt) stageReplace(
agent *Agent,
oboUser string,
scopeName string,
collectionName string,
key []byte,
value json.RawMessage,
cas Cas,
cb func(*TransactionGetResult, error),
) {
ecCb := func(result *TransactionGetResult, cerr *classifiedError) {
if cerr == nil {
cb(result, nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
switch cerr.Class {
case TransactionErrorClassFailExpiry:
t.setExpiryOvertimeAtomic()
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionExpired,
}))
case TransactionErrorClassFailDocNotFound:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrDocumentNotFound, "document not found during staging")),
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailDocAlreadyExists:
cerr.Class = TransactionErrorClassFailCasMismatch
fallthrough
case TransactionErrorClassFailCasMismatch:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailTransient:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailAmbiguous:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: false,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
case TransactionErrorClassFailHard:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
Reason: TransactionErrorReasonTransactionFailed,
}))
default:
cb(nil, t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: false,
Reason: TransactionErrorReasonTransactionFailed,
}))
}
}
t.checkExpiredAtomic(hookRemove, key, false, func(cerr *classifiedError) {
if cerr != nil {
ecCb(nil, cerr)
return
}
t.hooks.BeforeStagedReplace(key, func(err error) {
if err != nil {
ecCb(nil, classifyHookError(err))
return
}
stagedInfo := &transactionStagedMutation{
OpType: TransactionStagedMutationReplace,
Agent: agent,
OboUser: oboUser,
ScopeName: scopeName,
CollectionName: collectionName,
Key: key,
Staged: value,
}
var txnMeta jsonTxnXattr
txnMeta.ID.Transaction = t.transactionID
txnMeta.ID.Attempt = t.id
txnMeta.ATR.CollectionName = t.atrCollectionName
txnMeta.ATR.ScopeName = t.atrScopeName
txnMeta.ATR.BucketName = t.atrAgent.BucketName()
txnMeta.ATR.DocID = string(t.atrKey)
txnMeta.Operation.Type = jsonMutationReplace
txnMeta.Operation.Staged = stagedInfo.Staged
txnMeta.Restore = &jsonTxnXattrRestore{
OriginalCAS: "",
ExpiryTime: 0,
RevID: "",
}
txnMetaBytes, err := json.Marshal(txnMeta)
if err != nil {
ecCb(nil, classifyError(err))
return
}
deadline, duraTimeout := transactionsMutationTimeouts(t.keyValueTimeout, t.durabilityLevel)
_, err = stagedInfo.Agent.MutateIn(MutateInOptions{
ScopeName: stagedInfo.ScopeName,
CollectionName: stagedInfo.CollectionName,
Key: stagedInfo.Key,
Cas: cas,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "txn",
Flags: memd.SubdocFlagMkDirP | memd.SubdocFlagXattrPath,
Value: txnMetaBytes,
},
{
Op: memd.SubDocOpDictSet,
Path: "txn.op.crc32",
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
Value: crc32cMacro,
},
{
Op: memd.SubDocOpDictSet,
Path: "txn.restore.CAS",
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
Value: casMacro,
},
{
Op: memd.SubDocOpDictSet,
Path: "txn.restore.exptime",
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
Value: exptimeMacro,
},
{
Op: memd.SubDocOpDictSet,
Path: "txn.restore.revid",
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagExpandMacros,
Value: revidMacro,
},
},
Flags: memd.SubdocDocFlagAccessDeleted,
DurabilityLevel: transactionsDurabilityLevelToMemd(t.durabilityLevel),
DurabilityLevelTimeout: duraTimeout,
Deadline: deadline,
User: stagedInfo.OboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(nil, classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
stagedInfo.Cas = result.Cas
t.hooks.AfterStagedReplaceComplete(key, func(err error) {
if err != nil {
ecCb(nil, classifyHookError(err))
return
}
t.recordStagedMutation(stagedInfo, func() {
ecCb(&TransactionGetResult{
agent: stagedInfo.Agent,
oboUser: stagedInfo.OboUser,
scopeName: stagedInfo.ScopeName,
collectionName: stagedInfo.CollectionName,
key: stagedInfo.Key,
Value: stagedInfo.Staged,
Cas: stagedInfo.Cas,
Meta: nil,
}, nil)
})
})
})
if err != nil {
ecCb(nil, classifyError(err))
return
}
})
})
}
gocbcore-10.2.3/transactionattempt_rollback.go 0000664 0000000 0000000 00000024077 14417540156 0021525 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func (t *transactionAttempt) Rollback(cb TransactionRollbackCallback) error {
return t.rollback(func(err *TransactionOperationFailedError) {
if err != nil {
t.logger.logInfof(t.id, "Rollback failed")
t.ensureCleanUpRequest()
cb(err)
return
}
t.ensureCleanUpRequest()
cb(nil)
})
}
func (t *transactionAttempt) rollback(
cb func(*TransactionOperationFailedError),
) error {
t.logger.logInfof(t.id, "Rolling back")
t.waitForOpsAndLock(func(unlock func()) {
unlockAndCb := func(err *TransactionOperationFailedError) {
unlock()
cb(err)
}
err := t.checkCanRollbackLocked()
if err != nil {
unlockAndCb(err)
return
}
t.applyStateBits(transactionStateBitShouldNotCommit|transactionStateBitShouldNotRollback, 0)
if t.state == TransactionAttemptStateNothingWritten {
unlockAndCb(nil)
return
}
t.checkExpiredAtomic(hookRollback, []byte{}, true, func(cerr *classifiedError) {
if cerr != nil {
t.setExpiryOvertimeAtomic()
}
t.setATRAbortedLocked(func(err *TransactionOperationFailedError) {
if err != nil {
unlockAndCb(err)
return
}
t.state = TransactionAttemptStateAborted
go func() {
removeStagedMutation := func(
mutation *transactionStagedMutation,
unstageCb func(*TransactionOperationFailedError),
) {
switch mutation.OpType {
case TransactionStagedMutationInsert:
t.removeStagedInsert(*mutation, unstageCb)
case TransactionStagedMutationReplace:
fallthrough
case TransactionStagedMutationRemove:
t.removeStagedRemoveReplace(*mutation, unstageCb)
default:
unstageCb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrIllegalState, "unexpected staged mutation type")),
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
}
}
var mutErrs []*TransactionOperationFailedError
if !t.enableParallelUnstaging {
for _, mutation := range t.stagedMutations {
waitCh := make(chan struct{}, 1)
removeStagedMutation(mutation, func(err *TransactionOperationFailedError) {
if err != nil {
mutErrs = append(mutErrs, err)
waitCh <- struct{}{}
return
}
waitCh <- struct{}{}
})
<-waitCh
if len(mutErrs) > 0 {
break
}
}
} else {
type mutResult struct {
Err *TransactionOperationFailedError
}
numMutations := len(t.stagedMutations)
waitCh := make(chan mutResult, numMutations)
// Unlike the RFC we do insert and replace separately. We have a bug in gocbcore where subdocs
// will raise doc exists rather than a cas mismatch so we need to do these ops separately to tell
// how to handle that error.
for _, mutation := range t.stagedMutations {
removeStagedMutation(mutation, func(err *TransactionOperationFailedError) {
waitCh <- mutResult{
Err: err,
}
})
}
for i := 0; i < numMutations; i++ {
res := <-waitCh
if res.Err != nil {
mutErrs = append(mutErrs, res.Err)
continue
}
}
}
err = mergeOperationFailedErrors(mutErrs)
if err != nil {
unlockAndCb(err)
return
}
t.setATRRolledBackLocked(func(err *TransactionOperationFailedError) {
if err != nil {
unlockAndCb(err)
return
}
t.state = TransactionAttemptStateRolledBack
unlockAndCb(nil)
})
}()
})
})
})
return nil
}
func (t *transactionAttempt) removeStagedInsert(
mutation transactionStagedMutation,
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
if t.isExpiryOvertimeAtomic() {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "removing a staged insert failed during overtime")),
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
return
}
switch cerr.Class {
case TransactionErrorClassFailAmbiguous:
time.AfterFunc(3*time.Millisecond, func() {
t.removeStagedInsert(mutation, cb)
})
case TransactionErrorClassFailExpiry:
t.setExpiryOvertimeAtomic()
time.AfterFunc(3*time.Millisecond, func() {
t.removeStagedInsert(mutation, cb)
})
case TransactionErrorClassFailDocNotFound:
cb(nil)
return
case TransactionErrorClassFailPathNotFound:
cb(nil)
return
case TransactionErrorClassFailDocAlreadyExists:
cerr.Class = TransactionErrorClassFailCasMismatch
fallthrough
case TransactionErrorClassFailCasMismatch:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
case TransactionErrorClassFailHard:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
default:
time.AfterFunc(3*time.Millisecond, func() {
t.removeStagedInsert(mutation, cb)
})
}
}
t.checkExpiredAtomic(hookDeleteInserted, mutation.Key, true, func(cerr *classifiedError) {
if cerr != nil {
ecCb(cerr)
return
}
t.hooks.BeforeRollbackDeleteInserted(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
_, err = mutation.Agent.MutateIn(MutateInOptions{
ScopeName: mutation.ScopeName,
CollectionName: mutation.CollectionName,
Key: mutation.Key,
Cas: mutation.Cas,
Flags: memd.SubdocDocFlagAccessDeleted,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
Value: []byte{110, 117, 108, 108}, // null
},
{
Op: memd.SubDocOpDelete,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
},
User: mutation.OboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
for _, op := range result.Ops {
if op.Err != nil {
ecCb(classifyError(op.Err))
return
}
}
t.hooks.AfterRollbackDeleteInserted(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
ecCb(nil)
})
})
if err != nil {
ecCb(classifyError(err))
return
}
})
})
}
func (t *transactionAttempt) removeStagedRemoveReplace(
mutation transactionStagedMutation,
cb func(*TransactionOperationFailedError),
) {
ecCb := func(cerr *classifiedError) {
if cerr == nil {
cb(nil)
return
}
t.ReportResourceUnitsError(cerr.Source)
if t.isExpiryOvertimeAtomic() {
cb(t.operationFailed(operationFailedDef{
Cerr: classifyError(
wrapError(ErrAttemptExpired, "removing a staged remove or replace failed during overtime")),
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
return
}
switch cerr.Class {
case TransactionErrorClassFailAmbiguous:
time.AfterFunc(3*time.Millisecond, func() {
t.removeStagedRemoveReplace(mutation, cb)
})
case TransactionErrorClassFailExpiry:
t.setExpiryOvertimeAtomic()
time.AfterFunc(3*time.Millisecond, func() {
t.removeStagedRemoveReplace(mutation, cb)
})
case TransactionErrorClassFailPathNotFound:
cb(nil)
return
case TransactionErrorClassFailDocNotFound:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
case TransactionErrorClassFailDocAlreadyExists:
cerr.Class = TransactionErrorClassFailCasMismatch
fallthrough
case TransactionErrorClassFailCasMismatch:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
case TransactionErrorClassFailHard:
cb(t.operationFailed(operationFailedDef{
Cerr: cerr,
ShouldNotRetry: true,
ShouldNotRollback: true,
}))
default:
time.AfterFunc(3*time.Millisecond, func() {
t.removeStagedRemoveReplace(mutation, cb)
})
}
}
t.checkExpiredAtomic(hookRollbackDoc, mutation.Key, true, func(cerr *classifiedError) {
if cerr != nil {
ecCb(cerr)
return
}
t.hooks.BeforeDocRolledBack(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
_, err = mutation.Agent.MutateIn(MutateInOptions{
ScopeName: mutation.ScopeName,
CollectionName: mutation.CollectionName,
Key: mutation.Key,
Cas: mutation.Cas,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
Value: []byte{110, 117, 108, 108}, // null
},
{
Op: memd.SubDocOpDelete,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
},
User: mutation.OboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ecCb(classifyError(err))
return
}
t.ReportResourceUnits(result.Internal.ResourceUnits)
for _, op := range result.Ops {
if op.Err != nil {
ecCb(classifyError(op.Err))
return
}
}
t.hooks.AfterRollbackReplaceOrRemove(mutation.Key, func(err error) {
if err != nil {
ecCb(classifyHookError(err))
return
}
ecCb(nil)
})
})
if err != nil {
ecCb(classifyError(err))
return
}
})
})
}
gocbcore-10.2.3/transactions.go 0000664 0000000 0000000 00000033034 14417540156 0016431 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"time"
"github.com/google/uuid"
)
// TransactionsManager is the top level wrapper object for all transactions
// handling. It also manages the cleanup process in the background.
type TransactionsManager struct {
config TransactionsConfig
cleaner TransactionsCleaner
lostCleanup lostTransactionCleaner
}
// InitTransactions will initialize the transactions library and return a TransactionsManager
// object which can be used to perform transactions.
func InitTransactions(config *TransactionsConfig) (*TransactionsManager, error) {
logInfof("Initializing transactions: %s", config)
defaultConfig := &TransactionsConfig{
ExpirationTime: 10000 * time.Millisecond,
DurabilityLevel: TransactionDurabilityLevelMajority,
KeyValueTimeout: 2500 * time.Millisecond,
CleanupWindow: 60000 * time.Millisecond,
CleanupClientAttempts: true,
CleanupLostAttempts: true,
BucketAgentProvider: func(bucketName string) (*Agent, string, error) {
return nil, "", errors.New("no bucket agent provider was specified")
},
}
if config == nil {
config = defaultConfig
}
if config.ExpirationTime == 0 {
config.ExpirationTime = defaultConfig.ExpirationTime
}
if config.KeyValueTimeout == 0 {
config.KeyValueTimeout = defaultConfig.KeyValueTimeout
}
if config.CleanupWindow == 0 {
config.CleanupWindow = defaultConfig.CleanupWindow
}
if config.BucketAgentProvider == nil {
config.BucketAgentProvider = defaultConfig.BucketAgentProvider
}
if config.Internal.Hooks == nil {
config.Internal.Hooks = &TransactionDefaultHooks{}
}
if config.Internal.CleanUpHooks == nil {
config.Internal.CleanUpHooks = &TransactionDefaultCleanupHooks{}
}
if config.Internal.ClientRecordHooks == nil {
config.Internal.ClientRecordHooks = &TransactionDefaultClientRecordHooks{}
}
if config.CleanupQueueSize == 0 {
config.CleanupQueueSize = 100000
}
if config.Internal.NumATRs == 0 {
config.Internal.NumATRs = 1024
}
t := &TransactionsManager{
config: *config,
}
if config.CleanupClientAttempts {
t.cleaner = startCleanupThread(config)
} else {
t.cleaner = &noopTransactionsCleaner{}
}
if config.CleanupLostAttempts {
t.lostCleanup = startLostTransactionCleaner(config)
// We add the custom metadata location to the cleanup locations so that lost cleanup starts watching it
// immediately. Note that we don't do the same for the custom locations on TransactionOptions, this is because
// we know that that location will be used in a transaction.
if config.CustomATRLocation.Agent != nil {
t.addLostCleanupLocation(config.CustomATRLocation.Agent.BucketName(), config.CustomATRLocation.ScopeName, config.CustomATRLocation.CollectionName)
}
} else {
t.lostCleanup = &noopLostTransactionCleaner{}
}
return t, nil
}
// Config returns the config that was used during the initialization
// of this TransactionsManager object.
func (t *TransactionsManager) Config() TransactionsConfig {
return t.config
}
// BeginTransaction will begin a new transaction. The returned object can be used
// to begin a new attempt and subsequently perform operations before finally committing.
func (t *TransactionsManager) BeginTransaction(perConfig *TransactionOptions) (*Transaction, error) {
logDebugf("Beginning transaction: %s", perConfig)
transactionUUID := uuid.New().String()
expirationTime := t.config.ExpirationTime
durabilityLevel := t.config.DurabilityLevel
keyValueTimeout := t.config.KeyValueTimeout
customATRLocation := t.config.CustomATRLocation
bucketAgentProvider := t.config.BucketAgentProvider
hooks := t.config.Internal.Hooks
recordResourceUnit := noopResourceUnitCallback
var logger *internalTransactionLogWrapper
if perConfig != nil {
if perConfig.ExpirationTime != 0 {
expirationTime = perConfig.ExpirationTime
}
if perConfig.DurabilityLevel != TransactionDurabilityLevelUnknown {
durabilityLevel = perConfig.DurabilityLevel
}
if perConfig.KeyValueTimeout != 0 {
keyValueTimeout = perConfig.KeyValueTimeout
}
if perConfig.CustomATRLocation.Agent != nil {
customATRLocation = perConfig.CustomATRLocation
}
if perConfig.BucketAgentProvider != nil {
bucketAgentProvider = perConfig.BucketAgentProvider
}
if perConfig.Internal.Hooks != nil {
hooks = perConfig.Internal.Hooks
}
if perConfig.TransactionLogger == nil {
logger = newInternalTransactionLogger(transactionUUID, NewNoopTransactionLogger())
} else {
logger = newInternalTransactionLogger(transactionUUID, perConfig.TransactionLogger)
}
if perConfig.Internal.ResourceUnitCallback != nil {
recordResourceUnit = perConfig.Internal.ResourceUnitCallback
}
} else {
logger = newInternalTransactionLogger(transactionUUID, NewNoopTransactionLogger())
}
now := time.Now()
return &Transaction{
parent: t,
expiryTime: now.Add(expirationTime),
startTime: now,
durabilityLevel: durabilityLevel,
transactionID: transactionUUID,
keyValueTimeout: keyValueTimeout,
atrLocation: customATRLocation,
addCleanupRequest: t.addCleanupRequest,
hooks: hooks,
enableNonFatalGets: t.config.Internal.EnableNonFatalGets,
enableParallelUnstaging: t.config.Internal.EnableParallelUnstaging,
enableExplicitATRs: t.config.Internal.EnableExplicitATRs,
enableMutationCaching: t.config.Internal.EnableMutationCaching,
bucketAgentProvider: bucketAgentProvider,
addLostCleanupLocation: t.addLostCleanupLocation,
logger: logger,
recordResourceUnit: recordResourceUnit,
}, nil
}
// ResumeTransactionOptions specifies options which can be overridden for the resumed transaction.
type ResumeTransactionOptions struct {
// BucketAgentProvider provides a function which returns an agent for
// a particular bucket by name.
BucketAgentProvider TransactionsBucketAgentProviderFn
// TransactionLogger is the logger to use with this transaction.
TransactionLogger TransactionLogger
// Internal specifies a set of options for internal use.
// Internal: This should never be used and is not supported.
Internal struct {
ResourceUnitCallback func(result *ResourceUnitResult)
}
}
// ResumeTransactionAttempt allows the resumption of an existing transaction attempt
// which was previously serialized, potentially by a different transaction client.
func (t *TransactionsManager) ResumeTransactionAttempt(txnBytes []byte, options *ResumeTransactionOptions) (*Transaction, error) {
bucketAgentProvider := t.config.BucketAgentProvider
if options != nil {
if options.BucketAgentProvider != nil {
bucketAgentProvider = options.BucketAgentProvider
}
}
var txnData jsonSerializedAttempt
err := json.Unmarshal(txnBytes, &txnData)
if err != nil {
return nil, err
}
if txnData.ID.Transaction == "" {
return nil, errors.New("invalid txn data - no transaction id")
}
if txnData.Config.DurabilityLevel == "" {
return nil, errors.New("invalid txn data - no durability level")
}
if txnData.State.TimeLeftMs <= 0 {
return nil, errors.New("invalid txn data - time left must be greater than 0")
}
if txnData.Config.KeyValueTimeoutMs <= 0 {
return nil, errors.New("invalid txn data - operation timeout must be greater than 0")
}
if txnData.Config.NumAtrs <= 0 || txnData.Config.NumAtrs > 1024 {
return nil, errors.New("invalid txn data - num atrs must be greater than 0 and less than 1024")
}
var atrLocation TransactionATRLocation
if txnData.ATR.Bucket != "" && txnData.ATR.ID == "" {
// ATR references the specific ATR for this transaction.
foundAtrAgent, foundAtrOboUser, err := t.config.BucketAgentProvider(txnData.ATR.Bucket)
if err != nil {
return nil, err
}
atrLocation = TransactionATRLocation{
Agent: foundAtrAgent,
OboUser: foundAtrOboUser,
ScopeName: txnData.ATR.Scope,
CollectionName: txnData.ATR.Collection,
}
} else {
// No ATR information means its pending with no custom.
atrLocation = TransactionATRLocation{
Agent: nil,
OboUser: "",
ScopeName: "",
CollectionName: "",
}
}
transactionUUID := txnData.ID.Transaction
durabilityLevel, err := transactionDurabilityLevelFromString(txnData.Config.DurabilityLevel)
if err != nil {
return nil, err
}
expirationTime := time.Duration(txnData.State.TimeLeftMs) * time.Millisecond
keyValueTimeout := time.Duration(txnData.Config.KeyValueTimeoutMs) * time.Millisecond
var logger *internalTransactionLogWrapper
if options == nil || options.TransactionLogger == nil {
logger = newInternalTransactionLogger(transactionUUID, NewNoopTransactionLogger())
} else {
logger = newInternalTransactionLogger(transactionUUID, options.TransactionLogger)
}
recordResourceUnit := noopResourceUnitCallback
if options != nil && options.Internal.ResourceUnitCallback != nil {
recordResourceUnit = options.Internal.ResourceUnitCallback
}
now := time.Now()
txn := &Transaction{
parent: t,
expiryTime: now.Add(expirationTime),
startTime: now,
durabilityLevel: durabilityLevel,
transactionID: transactionUUID,
keyValueTimeout: keyValueTimeout,
atrLocation: atrLocation,
addCleanupRequest: t.addCleanupRequest,
hooks: t.config.Internal.Hooks,
enableNonFatalGets: t.config.Internal.EnableNonFatalGets,
enableParallelUnstaging: t.config.Internal.EnableParallelUnstaging,
enableExplicitATRs: t.config.Internal.EnableExplicitATRs,
enableMutationCaching: t.config.Internal.EnableMutationCaching,
bucketAgentProvider: bucketAgentProvider,
addLostCleanupLocation: t.addLostCleanupLocation,
logger: logger,
recordResourceUnit: recordResourceUnit,
}
err = txn.resumeAttempt(&txnData)
if err != nil {
return nil, err
}
return txn, nil
}
// Close will shut down this TransactionsManager object, shutting down all
// background tasks associated with it.
func (t *TransactionsManager) Close() error {
t.cleaner.Close()
t.lostCleanup.Close()
return nil
}
func (t *TransactionsManager) addCleanupRequest(req *TransactionsCleanupRequest) bool {
return t.cleaner.AddRequest(req)
}
func (t *TransactionsManager) addLostCleanupLocation(bucket, scope, collection string) {
if !t.config.CleanupLostAttempts {
return
}
go func() {
t.lostCleanup.AddATRLocation(TransactionLostATRLocation{
BucketName: bucket,
ScopeName: scope,
CollectionName: collection,
})
}()
}
// TransactionsManagerInternal exposes internal methods that are useful for testing and/or
// other forms of internal use.
type TransactionsManagerInternal struct {
parent *TransactionsManager
}
// Internal returns an TransactionsManagerInternal object which can be used for specialized
// internal use cases.
func (t *TransactionsManager) Internal() *TransactionsManagerInternal {
return &TransactionsManagerInternal{
parent: t,
}
}
// TransactionCreateGetResultOptions exposes options for the Internal CreateGetResult method.
type TransactionCreateGetResultOptions struct {
Agent *Agent
OboUser string
ScopeName string
CollectionName string
Key []byte
Cas Cas
Meta *TransactionMutableItemMeta
}
// CreateGetResult creates a false TransactionGetResult which can be used with Replace/Remove operations
// where the original TransactionGetResult is no longer available.
func (t *TransactionsManagerInternal) CreateGetResult(opts TransactionCreateGetResultOptions) *TransactionGetResult {
return &TransactionGetResult{
agent: opts.Agent,
oboUser: opts.OboUser,
scopeName: opts.ScopeName,
collectionName: opts.CollectionName,
key: opts.Key,
Meta: opts.Meta,
Value: nil,
Cas: opts.Cas,
}
}
// ForceCleanupQueue forces the transactions client cleanup queue to drain without waiting for expirations.
func (t *TransactionsManagerInternal) ForceCleanupQueue(cb func([]TransactionsCleanupAttempt)) {
t.parent.cleaner.ForceCleanupQueue(cb)
}
// CleanupQueueLength returns the current length of the client cleanup queue.
func (t *TransactionsManagerInternal) CleanupQueueLength() int32 {
return t.parent.cleaner.QueueLength()
}
// CleanupLocations returns the set of locations currently being watched by the lost transactions process.
func (t *TransactionsManagerInternal) CleanupLocations() []TransactionLostATRLocation {
return t.parent.lostCleanup.ATRLocations()
}
// LostCleanupGetAndResetResourceUnits returns the number of resource units used by the lost cleanup thread,
// and resets them.
func (t *TransactionsManagerInternal) LostCleanupGetAndResetResourceUnits() *TransactionResourceUnitResult {
return t.parent.lostCleanup.GetAndResetResourceUnits()
}
// CleanupThreadGetAndResetResourceUnits returns the number of resource units used by the standard cleanup thread,
// // and resets them.
func (t *TransactionsManagerInternal) CleanupThreadGetAndResetResourceUnits() *TransactionResourceUnitResult {
return t.parent.cleaner.GetAndResetResourceUnits()
}
gocbcore-10.2.3/transactions_atridlist.go 0000664 0000000 0000000 00000055015 14417540156 0020513 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
var transactionAtrIDList = []string{
"_txn:atr-0-#14",
"_txn:atr-1-#10b6",
"_txn:atr-2-#cc8",
"_txn:atr-3-#f08",
"_txn:atr-4-#c7",
"_txn:atr-5-#11a",
"_txn:atr-6-#a",
"_txn:atr-7-#2c4",
"_txn:atr-8-#4c",
"_txn:atr-9-#0",
"_txn:atr-10-#b8a",
"_txn:atr-11-#89e",
"_txn:atr-12-#ba8",
"_txn:atr-13-#129e",
"_txn:atr-14-#1429",
"_txn:atr-15-#39c8",
"_txn:atr-16-#1d8",
"_txn:atr-17-#2e8",
"_txn:atr-18-#1179",
"_txn:atr-19-#28",
"_txn:atr-20-#c68",
"_txn:atr-21-#fe8",
"_txn:atr-22-#59e8",
"_txn:atr-23-#a4",
"_txn:atr-24-#20c",
"_txn:atr-25-#11c",
"_txn:atr-26-#1b6",
"_txn:atr-27-#2c6",
"_txn:atr-28-#122a",
"_txn:atr-29-#123e",
"_txn:atr-30-#22",
"_txn:atr-31-#28d",
"_txn:atr-32-#183d",
"_txn:atr-33-#1861",
"_txn:atr-34-#b3",
"_txn:atr-35-#f19",
"_txn:atr-36-#b61",
"_txn:atr-37-#83d",
"_txn:atr-38-#b26",
"_txn:atr-39-#832",
"_txn:atr-40-#b94",
"_txn:atr-41-#4d",
"_txn:atr-42-#836",
"_txn:atr-43-#5",
"_txn:atr-44-#118",
"_txn:atr-45-#b6",
"_txn:atr-46-#1222",
"_txn:atr-47-#1236",
"_txn:atr-48-#199",
"_txn:atr-49-#289",
"_txn:atr-50-#21b",
"_txn:atr-51-#10b",
"_txn:atr-52-#bd",
"_txn:atr-53-#159",
"_txn:atr-54-#8a9",
"_txn:atr-55-#1206",
"_txn:atr-56-#bc1",
"_txn:atr-57-#8b5",
"_txn:atr-58-#c24",
"_txn:atr-59-#8bc",
"_txn:atr-60-#181",
"_txn:atr-61-#291",
"_txn:atr-62-#227",
"_txn:atr-63-#137",
"_txn:atr-64-#cd9",
"_txn:atr-65-#c9",
"_txn:atr-66-#929",
"_txn:atr-67-#18cd",
"_txn:atr-68-#8d",
"_txn:atr-69-#81c",
"_txn:atr-70-#888",
"_txn:atr-71-#121c",
"_txn:atr-72-#bbc",
"_txn:atr-73-#924",
"_txn:atr-74-#136",
"_txn:atr-75-#226",
"_txn:atr-76-#10b9",
"_txn:atr-77-#58b8",
"_txn:atr-78-#619",
"_txn:atr-79-#3b9",
"_txn:atr-80-#b1a",
"_txn:atr-81-#80e",
"_txn:atr-82-#8c0",
"_txn:atr-83-#bb4",
"_txn:atr-84-#1349",
"_txn:atr-85-#3878",
"_txn:atr-86-#3b8",
"_txn:atr-87-#618",
"_txn:atr-88-#2cd",
"_txn:atr-89-#1bd",
"_txn:atr-90-#1324",
"_txn:atr-91-#12bc",
"_txn:atr-92-#12f",
"_txn:atr-93-#23f",
"_txn:atr-94-#bc3",
"_txn:atr-95-#8b7",
"_txn:atr-96-#79",
"_txn:atr-97-#ff9",
"_txn:atr-98-#1bc7",
"_txn:atr-99-#8e8",
"_txn:atr-100-#bc0",
"_txn:atr-101-#8b4",
"_txn:atr-102-#81a",
"_txn:atr-103-#b0e",
"_txn:atr-104-#248",
"_txn:atr-105-#158",
"_txn:atr-106-#42",
"_txn:atr-107-#18b4",
"_txn:atr-108-#215",
"_txn:atr-109-#105",
"_txn:atr-110-#1217",
"_txn:atr-111-#1203",
"_txn:atr-112-#119",
"_txn:atr-113-#209",
"_txn:atr-114-#f1",
"_txn:atr-115-#8e9",
"_txn:atr-116-#b95",
"_txn:atr-117-#881",
"_txn:atr-118-#b9c",
"_txn:atr-119-#8de",
"_txn:atr-120-#2a3",
"_txn:atr-121-#1db",
"_txn:atr-122-#f8",
"_txn:atr-123-#22f",
"_txn:atr-124-#859",
"_txn:atr-125-#1d29",
"_txn:atr-126-#bc9",
"_txn:atr-127-#ab9",
"_txn:atr-128-#288",
"_txn:atr-129-#198",
"_txn:atr-130-#10c",
"_txn:atr-131-#21c",
"_txn:atr-132-#158",
"_txn:atr-133-#248",
"_txn:atr-134-#1805",
"_txn:atr-135-#8a8",
"_txn:atr-136-#56",
"_txn:atr-137-#bc0",
"_txn:atr-138-#8bb",
"_txn:atr-139-#bcf",
"_txn:atr-140-#b21",
"_txn:atr-141-#835",
"_txn:atr-142-#883",
"_txn:atr-143-#38",
"_txn:atr-144-#171",
"_txn:atr-145-#261",
"_txn:atr-146-#29f",
"_txn:atr-147-#4d",
"_txn:atr-148-#181d",
"_txn:atr-149-#1841",
"_txn:atr-150-#2b4",
"_txn:atr-151-#1c4",
"_txn:atr-152-#728",
"_txn:atr-153-#2a8",
"_txn:atr-154-#c3",
"_txn:atr-155-#848",
"_txn:atr-156-#b46",
"_txn:atr-157-#81c",
"_txn:atr-158-#b01",
"_txn:atr-159-#815",
"_txn:atr-160-#116",
"_txn:atr-161-#206",
"_txn:atr-162-#2cc",
"_txn:atr-163-#13",
"_txn:atr-164-#5",
"_txn:atr-165-#ad8",
"_txn:atr-166-#838",
"_txn:atr-167-#c0",
"_txn:atr-168-#8c7",
"_txn:atr-169-#bb3",
"_txn:atr-170-#b5",
"_txn:atr-171-#8",
"_txn:atr-172-#b21",
"_txn:atr-173-#835",
"_txn:atr-174-#1cd",
"_txn:atr-175-#2bd",
"_txn:atr-176-#171",
"_txn:atr-177-#261",
"_txn:atr-178-#136",
"_txn:atr-179-#b6",
"_txn:atr-180-#ba3",
"_txn:atr-181-#88d",
"_txn:atr-182-#4e",
"_txn:atr-183-#b2f",
"_txn:atr-184-#3c9",
"_txn:atr-185-#609",
"_txn:atr-186-#1f68",
"_txn:atr-187-#19e8",
"_txn:atr-188-#236",
"_txn:atr-189-#126",
"_txn:atr-190-#12c9",
"_txn:atr-191-#c2",
"_txn:atr-192-#1c2",
"_txn:atr-193-#2b2",
"_txn:atr-194-#b66",
"_txn:atr-195-#4f",
"_txn:atr-196-#12b4",
"_txn:atr-197-#868",
"_txn:atr-198-#39",
"_txn:atr-199-#f59",
"_txn:atr-200-#13c",
"_txn:atr-201-#4b",
"_txn:atr-202-#29e",
"_txn:atr-203-#d",
"_txn:atr-204-#1021",
"_txn:atr-205-#b0",
"_txn:atr-206-#a58",
"_txn:atr-207-#b48",
"_txn:atr-208-#4a",
"_txn:atr-209-#9c9",
"_txn:atr-210-#8cb",
"_txn:atr-211-#bbf",
"_txn:atr-212-#bb",
"_txn:atr-213-#1820",
"_txn:atr-214-#239",
"_txn:atr-215-#129",
"_txn:atr-216-#115",
"_txn:atr-217-#89",
"_txn:atr-218-#11c",
"_txn:atr-219-#20c",
"_txn:atr-220-#b2f",
"_txn:atr-221-#83b",
"_txn:atr-222-#88d",
"_txn:atr-223-#ba3",
"_txn:atr-224-#1b00",
"_txn:atr-225-#349",
"_txn:atr-226-#1458",
"_txn:atr-227-#39d9",
"_txn:atr-228-#c5",
"_txn:atr-229-#284",
"_txn:atr-230-#1f29",
"_txn:atr-231-#19a9",
"_txn:atr-232-#13c",
"_txn:atr-233-#22c",
"_txn:atr-234-#bb6",
"_txn:atr-235-#8c2",
"_txn:atr-236-#1b66",
"_txn:atr-237-#8b8",
"_txn:atr-238-#c29",
"_txn:atr-239-#fa9",
"_txn:atr-240-#8b7",
"_txn:atr-241-#bc3",
"_txn:atr-242-#b0f",
"_txn:atr-243-#81b",
"_txn:atr-244-#1168",
"_txn:atr-245-#5969",
"_txn:atr-246-#729",
"_txn:atr-247-#2a9",
"_txn:atr-248-#106",
"_txn:atr-249-#216",
"_txn:atr-250-#c",
"_txn:atr-251-#38",
"_txn:atr-252-#2a4",
"_txn:atr-253-#1de",
"_txn:atr-254-#80e",
"_txn:atr-255-#b1a",
"_txn:atr-256-#b11",
"_txn:atr-257-#805",
"_txn:atr-258-#b56",
"_txn:atr-259-#1b",
"_txn:atr-260-#12b",
"_txn:atr-261-#23b",
"_txn:atr-262-#28d",
"_txn:atr-263-#19d",
"_txn:atr-264-#ff9",
"_txn:atr-265-#c59",
"_txn:atr-266-#85",
"_txn:atr-267-#9c9",
"_txn:atr-268-#880",
"_txn:atr-269-#b94",
"_txn:atr-270-#59",
"_txn:atr-271-#1811",
"_txn:atr-272-#b88",
"_txn:atr-273-#a98",
"_txn:atr-274-#2e",
"_txn:atr-275-#128e",
"_txn:atr-276-#104",
"_txn:atr-277-#214",
"_txn:atr-278-#10b",
"_txn:atr-279-#21b",
"_txn:atr-280-#896",
"_txn:atr-281-#b82",
"_txn:atr-282-#b34",
"_txn:atr-283-#3b",
"_txn:atr-284-#18e8",
"_txn:atr-285-#1ad8",
"_txn:atr-286-#1124",
"_txn:atr-287-#92",
"_txn:atr-288-#261",
"_txn:atr-289-#171",
"_txn:atr-290-#1f9",
"_txn:atr-291-#459",
"_txn:atr-292-#1cb",
"_txn:atr-293-#2bb",
"_txn:atr-294-#b27",
"_txn:atr-295-#833",
"_txn:atr-296-#f79",
"_txn:atr-297-#cd9",
"_txn:atr-298-#848",
"_txn:atr-299-#1036",
"_txn:atr-300-#bc4",
"_txn:atr-301-#8b0",
"_txn:atr-302-#81e",
"_txn:atr-303-#b0a",
"_txn:atr-304-#1be",
"_txn:atr-305-#2ce",
"_txn:atr-306-#200",
"_txn:atr-307-#9d",
"_txn:atr-308-#1c9",
"_txn:atr-309-#2b9",
"_txn:atr-310-#231",
"_txn:atr-311-#121",
"_txn:atr-312-#1882",
"_txn:atr-313-#1896",
"_txn:atr-314-#21",
"_txn:atr-315-#ae9",
"_txn:atr-316-#b91",
"_txn:atr-317-#885",
"_txn:atr-318-#ba4",
"_txn:atr-319-#88c",
"_txn:atr-320-#1c7",
"_txn:atr-321-#2b7",
"_txn:atr-322-#21b",
"_txn:atr-323-#c5",
"_txn:atr-324-#b79",
"_txn:atr-325-#a69",
"_txn:atr-326-#8a9",
"_txn:atr-327-#16",
"_txn:atr-328-#816",
"_txn:atr-329-#b02",
"_txn:atr-330-#4b",
"_txn:atr-331-#1959",
"_txn:atr-332-#b8c",
"_txn:atr-333-#8a4",
"_txn:atr-334-#b0",
"_txn:atr-335-#20e",
"_txn:atr-336-#192",
"_txn:atr-337-#282",
"_txn:atr-338-#c",
"_txn:atr-339-#4a",
"_txn:atr-340-#13",
"_txn:atr-341-#141",
"_txn:atr-342-#1c5",
"_txn:atr-343-#2b5",
"_txn:atr-344-#bb9",
"_txn:atr-345-#ac9",
"_txn:atr-346-#849",
"_txn:atr-347-#180a",
"_txn:atr-348-#4d",
"_txn:atr-349-#8ba",
"_txn:atr-350-#878",
"_txn:atr-351-#12bf",
"_txn:atr-352-#86",
"_txn:atr-353-#b1e",
"_txn:atr-354-#29c",
"_txn:atr-355-#18c",
"_txn:atr-356-#2f",
"_txn:atr-357-#6f8",
"_txn:atr-358-#1bc2",
"_txn:atr-359-#1bb6",
"_txn:atr-360-#11d8",
"_txn:atr-361-#898",
"_txn:atr-362-#f58",
"_txn:atr-363-#cf8",
"_txn:atr-364-#5",
"_txn:atr-365-#210",
"_txn:atr-366-#2be",
"_txn:atr-367-#1ce",
"_txn:atr-368-#181",
"_txn:atr-369-#291",
"_txn:atr-370-#3a9",
"_txn:atr-371-#629",
"_txn:atr-372-#251",
"_txn:atr-373-#141",
"_txn:atr-374-#88b",
"_txn:atr-375-#b9f",
"_txn:atr-376-#bb9",
"_txn:atr-377-#bd",
"_txn:atr-378-#121d",
"_txn:atr-379-#928",
"_txn:atr-380-#639",
"_txn:atr-381-#529",
"_txn:atr-382-#1bc6",
"_txn:atr-383-#1bb2",
"_txn:atr-384-#1429",
"_txn:atr-385-#39c8",
"_txn:atr-386-#71",
"_txn:atr-387-#6d9",
"_txn:atr-388-#112",
"_txn:atr-389-#202",
"_txn:atr-390-#348",
"_txn:atr-391-#1014",
"_txn:atr-392-#1c6",
"_txn:atr-393-#2b6",
"_txn:atr-394-#b2c",
"_txn:atr-395-#85",
"_txn:atr-396-#b78",
"_txn:atr-397-#a68",
"_txn:atr-398-#1b8d",
"_txn:atr-399-#1ba3",
"_txn:atr-400-#22f",
"_txn:atr-401-#13f",
"_txn:atr-402-#1db",
"_txn:atr-403-#1",
"_txn:atr-404-#ab9",
"_txn:atr-405-#bc9",
"_txn:atr-406-#1923",
"_txn:atr-407-#19",
"_txn:atr-408-#b80",
"_txn:atr-409-#894",
"_txn:atr-410-#f38",
"_txn:atr-411-#e28",
"_txn:atr-412-#81e",
"_txn:atr-413-#b0a",
"_txn:atr-414-#2de",
"_txn:atr-415-#1a4",
"_txn:atr-416-#1149",
"_txn:atr-417-#5948",
"_txn:atr-418-#329",
"_txn:atr-419-#c3",
"_txn:atr-420-#888",
"_txn:atr-421-#1179",
"_txn:atr-422-#e58",
"_txn:atr-423-#f48",
"_txn:atr-424-#136",
"_txn:atr-425-#226",
"_txn:atr-426-#27",
"_txn:atr-427-#180",
"_txn:atr-428-#619",
"_txn:atr-429-#3b9",
"_txn:atr-430-#c9",
"_txn:atr-431-#1cb",
"_txn:atr-432-#22f",
"_txn:atr-433-#13f",
"_txn:atr-434-#de",
"_txn:atr-435-#bb3",
"_txn:atr-436-#ab9",
"_txn:atr-437-#bc9",
"_txn:atr-438-#938",
"_txn:atr-439-#12be",
"_txn:atr-440-#9d8",
"_txn:atr-441-#22",
"_txn:atr-442-#c28",
"_txn:atr-443-#fa8",
"_txn:atr-444-#292",
"_txn:atr-445-#8b",
"_txn:atr-446-#134",
"_txn:atr-447-#224",
"_txn:atr-448-#b9",
"_txn:atr-449-#108c",
"_txn:atr-450-#113",
"_txn:atr-451-#203",
"_txn:atr-452-#109",
"_txn:atr-453-#219",
"_txn:atr-454-#79a8",
"_txn:atr-455-#8d9",
"_txn:atr-456-#8cd",
"_txn:atr-457-#2c",
"_txn:atr-458-#8c2",
"_txn:atr-459-#bb6",
"_txn:atr-460-#2db",
"_txn:atr-461-#b3",
"_txn:atr-462-#12f",
"_txn:atr-463-#23f",
"_txn:atr-464-#895",
"_txn:atr-465-#b81",
"_txn:atr-466-#b37",
"_txn:atr-467-#823",
"_txn:atr-468-#e",
"_txn:atr-469-#1bdb",
"_txn:atr-470-#b06",
"_txn:atr-471-#812",
"_txn:atr-472-#f58",
"_txn:atr-473-#d8",
"_txn:atr-474-#12a9",
"_txn:atr-475-#39b8",
"_txn:atr-476-#2be",
"_txn:atr-477-#1ce",
"_txn:atr-478-#2b3",
"_txn:atr-479-#1c3",
"_txn:atr-480-#9e",
"_txn:atr-481-#b34",
"_txn:atr-482-#b82",
"_txn:atr-483-#1",
"_txn:atr-484-#35",
"_txn:atr-485-#1b95",
"_txn:atr-486-#13f9",
"_txn:atr-487-#38e8",
"_txn:atr-488-#8c",
"_txn:atr-489-#29b",
"_txn:atr-490-#209",
"_txn:atr-491-#119",
"_txn:atr-492-#2b",
"_txn:atr-493-#103",
"_txn:atr-494-#881",
"_txn:atr-495-#b95",
"_txn:atr-496-#80f",
"_txn:atr-497-#b1b",
"_txn:atr-498-#800",
"_txn:atr-499-#b14",
"_txn:atr-500-#cb",
"_txn:atr-501-#8f8",
"_txn:atr-502-#a18",
"_txn:atr-503-#b08",
"_txn:atr-504-#204",
"_txn:atr-505-#99",
"_txn:atr-506-#1ba",
"_txn:atr-507-#2ca",
"_txn:atr-508-#bd",
"_txn:atr-509-#149",
"_txn:atr-510-#193",
"_txn:atr-511-#283",
"_txn:atr-512-#6db",
"_txn:atr-513-#2f1",
"_txn:atr-514-#b25",
"_txn:atr-515-#831",
"_txn:atr-516-#5809",
"_txn:atr-517-#1008",
"_txn:atr-518-#b78",
"_txn:atr-519-#a68",
"_txn:atr-520-#3b9",
"_txn:atr-521-#97",
"_txn:atr-522-#1b24",
"_txn:atr-523-#1b30",
"_txn:atr-524-#8c1",
"_txn:atr-525-#76",
"_txn:atr-526-#b51",
"_txn:atr-527-#80d",
"_txn:atr-528-#94",
"_txn:atr-529-#b66",
"_txn:atr-530-#1b14",
"_txn:atr-531-#938",
"_txn:atr-532-#35",
"_txn:atr-533-#8ca",
"_txn:atr-534-#130",
"_txn:atr-535-#220",
"_txn:atr-536-#1a29",
"_txn:atr-537-#1839",
"_txn:atr-538-#529",
"_txn:atr-539-#639",
"_txn:atr-540-#18c0",
"_txn:atr-541-#18b4",
"_txn:atr-542-#3b",
"_txn:atr-543-#169",
"_txn:atr-544-#b71",
"_txn:atr-545-#82d",
"_txn:atr-546-#cc",
"_txn:atr-547-#3",
"_txn:atr-548-#1027",
"_txn:atr-549-#838",
"_txn:atr-550-#8b0",
"_txn:atr-551-#bc4",
"_txn:atr-552-#d28",
"_txn:atr-553-#e",
"_txn:atr-554-#2b08",
"_txn:atr-555-#44",
"_txn:atr-556-#146",
"_txn:atr-557-#256",
"_txn:atr-558-#101",
"_txn:atr-559-#211",
"_txn:atr-560-#c78",
"_txn:atr-561-#fd8",
"_txn:atr-562-#d",
"_txn:atr-563-#1c79",
"_txn:atr-564-#12e",
"_txn:atr-565-#23e",
"_txn:atr-566-#28c",
"_txn:atr-567-#19c",
"_txn:atr-568-#1816",
"_txn:atr-569-#1802",
"_txn:atr-570-#18d",
"_txn:atr-571-#29d",
"_txn:atr-572-#1221",
"_txn:atr-573-#1235",
"_txn:atr-574-#b49",
"_txn:atr-575-#a59",
"_txn:atr-576-#b0d",
"_txn:atr-577-#851",
"_txn:atr-578-#2d",
"_txn:atr-579-#816",
"_txn:atr-580-#1b7",
"_txn:atr-581-#2c7",
"_txn:atr-582-#20b",
"_txn:atr-583-#85",
"_txn:atr-584-#1876",
"_txn:atr-585-#182c",
"_txn:atr-586-#c69",
"_txn:atr-587-#fe9",
"_txn:atr-588-#806",
"_txn:atr-589-#b12",
"_txn:atr-590-#71",
"_txn:atr-591-#ff8",
"_txn:atr-592-#102b",
"_txn:atr-593-#9d8",
"_txn:atr-594-#198",
"_txn:atr-595-#288",
"_txn:atr-596-#182",
"_txn:atr-597-#292",
"_txn:atr-598-#18d",
"_txn:atr-599-#3c",
"_txn:atr-600-#48",
"_txn:atr-601-#898",
"_txn:atr-602-#f58",
"_txn:atr-603-#cf8",
"_txn:atr-604-#100",
"_txn:atr-605-#5",
"_txn:atr-606-#2be",
"_txn:atr-607-#1ce",
"_txn:atr-608-#1bb7",
"_txn:atr-609-#cf",
"_txn:atr-610-#2cf",
"_txn:atr-611-#1bf",
"_txn:atr-612-#1046",
"_txn:atr-613-#14",
"_txn:atr-614-#a39",
"_txn:atr-615-#b29",
"_txn:atr-616-#b15",
"_txn:atr-617-#801",
"_txn:atr-618-#b1c",
"_txn:atr-619-#846",
"_txn:atr-620-#12f",
"_txn:atr-621-#23f",
"_txn:atr-622-#2db",
"_txn:atr-623-#1a3",
"_txn:atr-624-#82",
"_txn:atr-625-#823",
"_txn:atr-626-#895",
"_txn:atr-627-#b81",
"_txn:atr-628-#cc8",
"_txn:atr-629-#f08",
"_txn:atr-630-#8f6",
"_txn:atr-631-#bcc",
"_txn:atr-632-#9e",
"_txn:atr-633-#898",
"_txn:atr-634-#668",
"_txn:atr-635-#3e8",
"_txn:atr-636-#35",
"_txn:atr-637-#210",
"_txn:atr-638-#10f",
"_txn:atr-639-#21f",
"_txn:atr-640-#f39",
"_txn:atr-641-#c3",
"_txn:atr-642-#1003",
"_txn:atr-643-#969",
"_txn:atr-644-#28f",
"_txn:atr-645-#10",
"_txn:atr-646-#161",
"_txn:atr-647-#271",
"_txn:atr-648-#3959",
"_txn:atr-649-#b5",
"_txn:atr-650-#266",
"_txn:atr-651-#176",
"_txn:atr-652-#19a9",
"_txn:atr-653-#46",
"_txn:atr-654-#ab8",
"_txn:atr-655-#bc8",
"_txn:atr-656-#828",
"_txn:atr-657-#12d8",
"_txn:atr-658-#f19",
"_txn:atr-659-#cb9",
"_txn:atr-660-#1458",
"_txn:atr-661-#39d9",
"_txn:atr-662-#6d8",
"_txn:atr-663-#378",
"_txn:atr-664-#9a",
"_txn:atr-665-#8c2",
"_txn:atr-666-#846",
"_txn:atr-667-#b1c",
"_txn:atr-668-#c29",
"_txn:atr-669-#fa9",
"_txn:atr-670-#83f",
"_txn:atr-671-#b2b",
"_txn:atr-672-#f39",
"_txn:atr-673-#e29",
"_txn:atr-674-#12c7",
"_txn:atr-675-#12b3",
"_txn:atr-676-#d",
"_txn:atr-677-#19f",
"_txn:atr-678-#280",
"_txn:atr-679-#190",
"_txn:atr-680-#b51",
"_txn:atr-681-#80d",
"_txn:atr-682-#8c1",
"_txn:atr-683-#bb5",
"_txn:atr-684-#2ce8",
"_txn:atr-685-#39",
"_txn:atr-686-#3b9",
"_txn:atr-687-#619",
"_txn:atr-688-#a9",
"_txn:atr-689-#2938",
"_txn:atr-690-#296",
"_txn:atr-691-#186",
"_txn:atr-692-#10f8",
"_txn:atr-693-#58f9",
"_txn:atr-694-#f68",
"_txn:atr-695-#ce8",
"_txn:atr-696-#b36",
"_txn:atr-697-#1c",
"_txn:atr-698-#b71",
"_txn:atr-699-#82d",
"_txn:atr-700-#18b1",
"_txn:atr-701-#ba9",
"_txn:atr-702-#5948",
"_txn:atr-703-#879",
"_txn:atr-704-#251",
"_txn:atr-705-#141",
"_txn:atr-706-#1c5",
"_txn:atr-707-#2b5",
"_txn:atr-708-#3e",
"_txn:atr-709-#1016",
"_txn:atr-710-#1de",
"_txn:atr-711-#2a4",
"_txn:atr-712-#2b8",
"_txn:atr-713-#1c8",
"_txn:atr-714-#878",
"_txn:atr-715-#1a88",
"_txn:atr-716-#82c",
"_txn:atr-717-#b76",
"_txn:atr-718-#825",
"_txn:atr-719-#b31",
"_txn:atr-720-#358",
"_txn:atr-721-#ba",
"_txn:atr-722-#1c48",
"_txn:atr-723-#2879",
"_txn:atr-724-#80a",
"_txn:atr-725-#43",
"_txn:atr-726-#bb0",
"_txn:atr-727-#8c4",
"_txn:atr-728-#1833",
"_txn:atr-729-#9f9",
"_txn:atr-730-#8b3",
"_txn:atr-731-#bc7",
"_txn:atr-732-#8c9",
"_txn:atr-733-#15",
"_txn:atr-734-#269",
"_txn:atr-735-#179",
"_txn:atr-736-#251",
"_txn:atr-737-#a4",
"_txn:atr-738-#216",
"_txn:atr-739-#106",
"_txn:atr-740-#192",
"_txn:atr-741-#282",
"_txn:atr-742-#234",
"_txn:atr-743-#124",
"_txn:atr-744-#a28",
"_txn:atr-745-#b38",
"_txn:atr-746-#1871",
"_txn:atr-747-#90",
"_txn:atr-748-#871",
"_txn:atr-749-#b2d",
"_txn:atr-750-#1294",
"_txn:atr-751-#d",
"_txn:atr-752-#b79",
"_txn:atr-753-#a69",
"_txn:atr-754-#1841",
"_txn:atr-755-#71",
"_txn:atr-756-#1c7",
"_txn:atr-757-#2b7",
"_txn:atr-758-#1ca",
"_txn:atr-759-#2ba",
"_txn:atr-760-#8cf",
"_txn:atr-761-#bbb",
"_txn:atr-762-#b17",
"_txn:atr-763-#78",
"_txn:atr-764-#2979",
"_txn:atr-765-#1b48",
"_txn:atr-766-#1e9",
"_txn:atr-767-#99",
"_txn:atr-768-#659",
"_txn:atr-769-#3f9",
"_txn:atr-770-#c3",
"_txn:atr-771-#1c3",
"_txn:atr-772-#2c9",
"_txn:atr-773-#1b9",
"_txn:atr-774-#10",
"_txn:atr-775-#108d",
"_txn:atr-776-#b41",
"_txn:atr-777-#81d",
"_txn:atr-778-#b5",
"_txn:atr-779-#812",
"_txn:atr-780-#1b3",
"_txn:atr-781-#2c3",
"_txn:atr-782-#20f",
"_txn:atr-783-#4f",
"_txn:atr-784-#bcb",
"_txn:atr-785-#8bf",
"_txn:atr-786-#813",
"_txn:atr-787-#b07",
"_txn:atr-788-#1929",
"_txn:atr-789-#968",
"_txn:atr-790-#822",
"_txn:atr-791-#b4",
"_txn:atr-792-#ce8",
"_txn:atr-793-#f68",
"_txn:atr-794-#1023",
"_txn:atr-795-#1037",
"_txn:atr-796-#186",
"_txn:atr-797-#296",
"_txn:atr-798-#1db",
"_txn:atr-799-#c2",
"_txn:atr-800-#729",
"_txn:atr-801-#c1",
"_txn:atr-802-#1b98",
"_txn:atr-803-#2839",
"_txn:atr-804-#b0f",
"_txn:atr-805-#12",
"_txn:atr-806-#8b7",
"_txn:atr-807-#bc3",
"_txn:atr-808-#828",
"_txn:atr-809-#b7",
"_txn:atr-810-#bb4",
"_txn:atr-811-#8c0",
"_txn:atr-812-#120e",
"_txn:atr-813-#44",
"_txn:atr-814-#618",
"_txn:atr-815-#3b8",
"_txn:atr-816-#1891",
"_txn:atr-817-#79",
"_txn:atr-818-#1b9",
"_txn:atr-819-#2c9",
"_txn:atr-820-#18cf",
"_txn:atr-821-#c8",
"_txn:atr-822-#a78",
"_txn:atr-823-#b68",
"_txn:atr-824-#9c",
"_txn:atr-825-#13e",
"_txn:atr-826-#18c",
"_txn:atr-827-#29c",
"_txn:atr-828-#239",
"_txn:atr-829-#129",
"_txn:atr-830-#1c1",
"_txn:atr-831-#2b1",
"_txn:atr-832-#188d",
"_txn:atr-833-#18a3",
"_txn:atr-834-#b69",
"_txn:atr-835-#a79",
"_txn:atr-836-#b3b",
"_txn:atr-837-#82f",
"_txn:atr-838-#b34",
"_txn:atr-839-#820",
"_txn:atr-840-#b86",
"_txn:atr-841-#892",
"_txn:atr-842-#824",
"_txn:atr-843-#b30",
"_txn:atr-844-#678",
"_txn:atr-845-#3d8",
"_txn:atr-846-#1292",
"_txn:atr-847-#1286",
"_txn:atr-848-#1d9",
"_txn:atr-849-#2e9",
"_txn:atr-850-#241",
"_txn:atr-851-#151",
"_txn:atr-852-#529",
"_txn:atr-853-#639",
"_txn:atr-854-#959",
"_txn:atr-855-#12ce",
"_txn:atr-856-#bb3",
"_txn:atr-857-#8c7",
"_txn:atr-858-#bbe",
"_txn:atr-859-#b8",
"_txn:atr-860-#193",
"_txn:atr-861-#283",
"_txn:atr-862-#235",
"_txn:atr-863-#14",
"_txn:atr-864-#1009",
"_txn:atr-865-#829",
"_txn:atr-866-#ae9",
"_txn:atr-867-#9f",
"_txn:atr-868-#b2a",
"_txn:atr-869-#83e",
"_txn:atr-870-#af8",
"_txn:atr-871-#d58",
"_txn:atr-872-#b86",
"_txn:atr-873-#892",
"_txn:atr-874-#23",
"_txn:atr-875-#200",
"_txn:atr-876-#678",
"_txn:atr-877-#3d8",
"_txn:atr-878-#1893",
"_txn:atr-879-#1887",
"_txn:atr-880-#b76",
"_txn:atr-881-#82c",
"_txn:atr-882-#21",
"_txn:atr-883-#b8a",
"_txn:atr-884-#1c8",
"_txn:atr-885-#2b8",
"_txn:atr-886-#8a",
"_txn:atr-887-#1bc1",
"_txn:atr-888-#291",
"_txn:atr-889-#181",
"_txn:atr-890-#629",
"_txn:atr-891-#9d",
"_txn:atr-892-#141",
"_txn:atr-893-#251",
"_txn:atr-894-#b9f",
"_txn:atr-895-#34",
"_txn:atr-896-#10d",
"_txn:atr-897-#21d",
"_txn:atr-898-#102",
"_txn:atr-899-#212",
"_txn:atr-900-#32",
"_txn:atr-901-#6a8",
"_txn:atr-902-#59c8",
"_txn:atr-903-#11c9",
"_txn:atr-904-#b02",
"_txn:atr-905-#816",
"_txn:atr-906-#8bc",
"_txn:atr-907-#c24",
"_txn:atr-908-#24",
"_txn:atr-909-#9a9",
"_txn:atr-910-#891",
"_txn:atr-911-#b85",
"_txn:atr-912-#59",
"_txn:atr-913-#b0b",
"_txn:atr-914-#28d",
"_txn:atr-915-#19d",
"_txn:atr-916-#12b0",
"_txn:atr-917-#12c4",
"_txn:atr-918-#298",
"_txn:atr-919-#188",
"_txn:atr-920-#be9",
"_txn:atr-921-#c0",
"_txn:atr-922-#839",
"_txn:atr-923-#48a9",
"_txn:atr-924-#117",
"_txn:atr-925-#13",
"_txn:atr-926-#2cb",
"_txn:atr-927-#1bb",
"_txn:atr-928-#168",
"_txn:atr-929-#278",
"_txn:atr-930-#2ba",
"_txn:atr-931-#0",
"_txn:atr-932-#328",
"_txn:atr-933-#6a8",
"_txn:atr-934-#1289",
"_txn:atr-935-#3828",
"_txn:atr-936-#9",
"_txn:atr-937-#86",
"_txn:atr-938-#b0d",
"_txn:atr-939-#851",
"_txn:atr-940-#88d",
"_txn:atr-941-#ba3",
"_txn:atr-942-#b2f",
"_txn:atr-943-#83b",
"_txn:atr-944-#291",
"_txn:atr-945-#181",
"_txn:atr-946-#9d",
"_txn:atr-947-#227",
"_txn:atr-948-#1349",
"_txn:atr-949-#3878",
"_txn:atr-950-#106",
"_txn:atr-951-#216",
"_txn:atr-952-#558",
"_txn:atr-953-#648",
"_txn:atr-954-#928",
"_txn:atr-955-#21",
"_txn:atr-956-#8ba",
"_txn:atr-957-#bce",
"_txn:atr-958-#8b7",
"_txn:atr-959-#bc3",
"_txn:atr-960-#28c",
"_txn:atr-961-#19c",
"_txn:atr-962-#c5",
"_txn:atr-963-#23e",
"_txn:atr-964-#9a8",
"_txn:atr-965-#1bc8",
"_txn:atr-966-#16",
"_txn:atr-967-#fd8",
"_txn:atr-968-#b25",
"_txn:atr-969-#831",
"_txn:atr-970-#8b9",
"_txn:atr-971-#4b",
"_txn:atr-972-#8c3",
"_txn:atr-973-#bb7",
"_txn:atr-974-#22b",
"_txn:atr-975-#b0",
"_txn:atr-976-#2bf",
"_txn:atr-977-#1cf",
"_txn:atr-978-#4a",
"_txn:atr-979-#1c0",
"_txn:atr-980-#801",
"_txn:atr-981-#b15",
"_txn:atr-982-#bf1",
"_txn:atr-983-#b2",
"_txn:atr-984-#2939",
"_txn:atr-985-#1b08",
"_txn:atr-986-#219",
"_txn:atr-987-#109",
"_txn:atr-988-#1b0",
"_txn:atr-989-#2c0",
"_txn:atr-990-#14",
"_txn:atr-991-#138",
"_txn:atr-992-#210",
"_txn:atr-993-#100",
"_txn:atr-994-#9f",
"_txn:atr-995-#b96",
"_txn:atr-996-#898",
"_txn:atr-997-#1f08",
"_txn:atr-998-#aa9",
"_txn:atr-999-#d29",
"_txn:atr-1000-#858",
"_txn:atr-1001-#1249",
"_txn:atr-1002-#bc8",
"_txn:atr-1003-#ab8",
"_txn:atr-1004-#29a",
"_txn:atr-1005-#18a",
"_txn:atr-1006-#59",
"_txn:atr-1007-#266",
"_txn:atr-1008-#2",
"_txn:atr-1009-#141",
"_txn:atr-1010-#59",
"_txn:atr-1011-#2d9",
"_txn:atr-1012-#181",
"_txn:atr-1013-#5",
"_txn:atr-1014-#b17",
"_txn:atr-1015-#803",
"_txn:atr-1016-#f49",
"_txn:atr-1017-#e59",
"_txn:atr-1018-#878",
"_txn:atr-1019-#1006",
"_txn:atr-1020-#249",
"_txn:atr-1021-#159",
"_txn:atr-1022-#cb",
"_txn:atr-1023-#10c2",
}
gocbcore-10.2.3/transactions_cleanup.go 0000664 0000000 0000000 00000065004 14417540156 0020142 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
// TransactionsCleanupRequest represents a complete transaction attempt that requires cleanup.
// Internal: This should never be used and is not supported.
type TransactionsCleanupRequest struct {
AttemptID string
AtrID []byte
AtrCollectionName string
AtrScopeName string
AtrBucketName string
Inserts []TransactionsDocRecord
Replaces []TransactionsDocRecord
Removes []TransactionsDocRecord
State TransactionAttemptState
ForwardCompat map[string][]TransactionForwardCompatibilityEntry
DurabilityLevel TransactionDurabilityLevel
Age time.Duration
}
func (cr *TransactionsCleanupRequest) String() string {
if isLogRedactionLevelFull() || isLogRedactionLevelPartial() {
return cr.redacted().(string)
}
return fmt.Sprintf(
"bucket: %s, collection: %s, scope: %s, atr: %s, attempt: %s, state: %s, age: %s",
cr.AtrBucketName,
cr.AtrCollectionName,
cr.AtrScopeName,
cr.AtrID,
cr.AttemptID,
cr.State,
cr.Age,
)
}
func (cr *TransactionsCleanupRequest) redacted() interface{} {
return fmt.Sprintf(
"bucket: %s, collection: %s, scope: %s, atr: %s, attempt: %s, state: %s, age: %s",
redactMetaData(cr.AtrBucketName),
redactMetaData(cr.AtrCollectionName),
redactMetaData(cr.AtrScopeName),
cr.AtrID,
cr.AttemptID,
cr.State,
cr.Age,
)
}
// TransactionsDocRecord represents an individual document operation requiring cleanup.
// Internal: This should never be used and is not supported.
type TransactionsDocRecord struct {
CollectionName string
ScopeName string
BucketName string
ID []byte
}
// TransactionsCleanupAttempt represents the result of running cleanup for a transaction attempt.
// Internal: This should never be used and is not supported.
type TransactionsCleanupAttempt struct {
Success bool
IsReqular bool
AttemptID string
AtrID []byte
AtrCollectionName string
AtrScopeName string
AtrBucketName string
Request *TransactionsCleanupRequest
}
func (ca TransactionsCleanupAttempt) String() string {
return fmt.Sprintf("bucket: %s, collection: %s, scope: %s, atr: %s, attempt: %s", ca.AtrBucketName, ca.AtrCollectionName,
ca.AtrScopeName, ca.AtrID, ca.AttemptID)
}
// TransactionsCleaner is responsible for performing cleanup of completed transactions.
// Internal: This should never be used and is not supported.
type TransactionsCleaner interface {
AddRequest(req *TransactionsCleanupRequest) bool
PopRequest() *TransactionsCleanupRequest
ForceCleanupQueue(cb func([]TransactionsCleanupAttempt))
QueueLength() int32
CleanupAttempt(atrAgent *Agent, atrOboUser string, req *TransactionsCleanupRequest, regular bool, cb func(attempt TransactionsCleanupAttempt))
Close()
GetAndResetResourceUnits() *TransactionResourceUnitResult
}
// NewTransactionsCleaner returns a TransactionsCleaner implementation.
// Internal: This should never be used and is not supported.
func NewTransactionsCleaner(config *TransactionsConfig) TransactionsCleaner {
return newStdCleaner(config)
}
type noopTransactionsCleaner struct {
}
func (nc *noopTransactionsCleaner) AddRequest(req *TransactionsCleanupRequest) bool {
return true
}
func (nc *noopTransactionsCleaner) PopRequest() *TransactionsCleanupRequest {
return nil
}
func (nc *noopTransactionsCleaner) ForceCleanupQueue(cb func([]TransactionsCleanupAttempt)) {
cb([]TransactionsCleanupAttempt{})
}
func (nc *noopTransactionsCleaner) QueueLength() int32 {
return 0
}
func (nc *noopTransactionsCleaner) GetAndResetResourceUnits() *TransactionResourceUnitResult {
return nil
}
func (nc *noopTransactionsCleaner) CleanupAttempt(atrAgent *Agent, atrOboUser string, req *TransactionsCleanupRequest, regular bool, cb func(attempt TransactionsCleanupAttempt)) {
cb(TransactionsCleanupAttempt{})
}
func (nc *noopTransactionsCleaner) Close() {}
type stdTransactionsCleaner struct {
hooks TransactionCleanUpHooks
qSize uint32
q chan *TransactionsCleanupRequest
stop chan struct{}
bucketAgentProvider TransactionsBucketAgentProviderFn
keyValueTimeout time.Duration
durabilityLevel TransactionDurabilityLevel
numResourceUnitOps uint32
readUnits uint32
writeUnits uint32
}
func newStdCleaner(config *TransactionsConfig) *stdTransactionsCleaner {
return &stdTransactionsCleaner{
hooks: config.Internal.CleanUpHooks,
qSize: config.CleanupQueueSize,
stop: make(chan struct{}),
bucketAgentProvider: config.BucketAgentProvider,
q: make(chan *TransactionsCleanupRequest, config.CleanupQueueSize),
keyValueTimeout: config.KeyValueTimeout,
durabilityLevel: config.DurabilityLevel,
}
}
func startCleanupThread(config *TransactionsConfig) *stdTransactionsCleaner {
cleaner := newStdCleaner(config)
// No point in running this if we can't get agents.
if config.BucketAgentProvider != nil {
go cleaner.processQ()
}
return cleaner
}
func (c *stdTransactionsCleaner) AddRequest(req *TransactionsCleanupRequest) bool {
select {
case c.q <- req:
// success!
default:
logDebugf("Not queueing request for: %s, limit size reached",
req.String())
}
return true
}
func (c *stdTransactionsCleaner) PopRequest() *TransactionsCleanupRequest {
select {
case req := <-c.q:
return req
default:
return nil
}
}
func (c *stdTransactionsCleaner) stealAllRequests() []*TransactionsCleanupRequest {
reqs := make([]*TransactionsCleanupRequest, 0, len(c.q))
for {
select {
case req := <-c.q:
reqs = append(reqs, req)
default:
return reqs
}
}
}
func (c *stdTransactionsCleaner) updateResourceUnits(units *ResourceUnitResult) {
if units == nil {
return
}
atomic.AddUint32(&c.numResourceUnitOps, 1)
atomic.AddUint32(&c.readUnits, uint32(units.ReadUnits))
atomic.AddUint32(&c.writeUnits, uint32(units.WriteUnits))
}
func (c *stdTransactionsCleaner) updateResourceUnitsError(err error) {
if err == nil {
return
}
var kerr *KeyValueError
if errors.As(err, &kerr) {
c.updateResourceUnits(kerr.Internal.ResourceUnits)
}
}
func (c *stdTransactionsCleaner) GetAndResetResourceUnits() *TransactionResourceUnitResult {
numOps := atomic.SwapUint32(&c.numResourceUnitOps, 0)
if numOps == 0 {
return nil
}
return &TransactionResourceUnitResult{
NumOps: numOps,
ReadUnits: atomic.SwapUint32(&c.readUnits, 0),
WriteUnits: atomic.SwapUint32(&c.writeUnits, 0),
}
}
// Used only for tests
func (c *stdTransactionsCleaner) ForceCleanupQueue(cb func([]TransactionsCleanupAttempt)) {
reqs := c.stealAllRequests()
if len(reqs) == 0 {
cb(nil)
return
}
results := make([]TransactionsCleanupAttempt, 0, len(reqs))
var l sync.Mutex
handler := func(attempt TransactionsCleanupAttempt) {
l.Lock()
defer l.Unlock()
results = append(results, attempt)
if len(results) == len(reqs) {
cb(results)
}
}
for _, req := range reqs {
agent, oboUser, err := c.bucketAgentProvider(req.AtrBucketName)
if err != nil {
handler(TransactionsCleanupAttempt{
Success: false,
IsReqular: false,
AttemptID: req.AttemptID,
AtrID: req.AtrID,
AtrCollectionName: req.AtrCollectionName,
AtrScopeName: req.AtrScopeName,
AtrBucketName: req.AtrBucketName,
Request: req,
})
continue
}
c.CleanupAttempt(agent, oboUser, req, true, func(attempt TransactionsCleanupAttempt) {
handler(attempt)
})
}
}
// Used only for tests
func (c *stdTransactionsCleaner) QueueLength() int32 {
return int32(len(c.q))
}
// Used only for tests
func (c *stdTransactionsCleaner) Close() {
close(c.stop)
}
func (c *stdTransactionsCleaner) processQ() {
logDebugf("Starting cleanup for %p", c)
for {
select {
case req := <-c.q:
agent, oboUser, err := c.bucketAgentProvider(req.AtrBucketName)
if err != nil {
logDebugf("Failed to get agent for request: %s, err: %v", req.String(), err)
return
}
logSchedf("Running cleanup for request: %s", req.String())
waitCh := make(chan struct{}, 1)
c.CleanupAttempt(agent, oboUser, req, true, func(attempt TransactionsCleanupAttempt) {
if !attempt.Success {
logDebugf("Cleanup attempt failed for entry: %s",
attempt.String())
}
waitCh <- struct{}{}
})
<-waitCh
case <-c.stop:
return
}
}
}
func (c *stdTransactionsCleaner) checkForwardCompatability(
stage forwardCompatStage,
fc map[string][]TransactionForwardCompatibilityEntry,
cb func(error),
) {
isCompat, _, _, err := checkForwardCompatability(stage, fc)
if err != nil {
cb(err)
return
}
if !isCompat {
cb(ErrForwardCompatibilityFailure)
return
}
cb(nil)
}
func (c *stdTransactionsCleaner) CleanupAttempt(atrAgent *Agent, atrOboUser string, req *TransactionsCleanupRequest, regular bool, cb func(attempt TransactionsCleanupAttempt)) {
beforeCb := func(stage string, attempt TransactionsCleanupAttempt) {
if attempt.Success {
cb(attempt)
return
}
logWarnf("Cleanup attempt %v with %p failed at %s check", req, c, stage)
if req.Age > 2*time.Hour {
logWarnf("Cleanup request is %s old which could indicate a serious error - please raise with support.", req.Age)
}
cb(attempt)
}
logSchedf("Cleaning up attempt %s with %p", req.AttemptID, c)
c.checkForwardCompatability(forwardCompatStageGetsCleanupEntry, req.ForwardCompat, func(err error) {
if err != nil {
beforeCb("forward compatability", TransactionsCleanupAttempt{
Success: false,
IsReqular: regular,
AttemptID: req.AttemptID,
AtrID: req.AtrID,
AtrCollectionName: req.AtrCollectionName,
AtrScopeName: req.AtrScopeName,
AtrBucketName: req.AtrBucketName,
Request: req,
})
return
}
c.cleanupDocs(req, func(err error) {
if err != nil {
beforeCb("cleanup docs", TransactionsCleanupAttempt{
Success: false,
IsReqular: regular,
AttemptID: req.AttemptID,
AtrID: req.AtrID,
AtrCollectionName: req.AtrCollectionName,
AtrScopeName: req.AtrScopeName,
AtrBucketName: req.AtrBucketName,
Request: req,
})
return
}
c.cleanupATR(atrAgent, atrOboUser, req, func(err error) {
success := true
if err != nil {
success = false
}
beforeCb("cleanup atr", TransactionsCleanupAttempt{
Success: success,
IsReqular: regular,
AttemptID: req.AttemptID,
AtrID: req.AtrID,
AtrCollectionName: req.AtrCollectionName,
AtrScopeName: req.AtrScopeName,
AtrBucketName: req.AtrBucketName,
Request: req,
})
})
})
})
}
func (c *stdTransactionsCleaner) cleanupATR(agent *Agent, oboUser string, req *TransactionsCleanupRequest, cb func(error)) {
c.hooks.BeforeATRRemove(req.AtrID, func(err error) {
if err != nil {
if errors.Is(err, ErrPathNotFound) {
cb(nil)
return
}
cb(err)
return
}
var specs []SubDocOp
if req.State == TransactionAttemptStatePending {
specs = append(specs, SubDocOp{
Op: memd.SubDocOpDictAdd,
Value: []byte{110, 117, 108, 108},
Path: "attempts." + req.AttemptID + ".p",
Flags: memd.SubdocFlagXattrPath,
})
}
specs = append(specs, SubDocOp{
Op: memd.SubDocOpDelete,
Path: "attempts." + req.AttemptID,
Flags: memd.SubdocFlagXattrPath,
})
if req.DurabilityLevel == TransactionDurabilityLevelUnknown {
req.DurabilityLevel = c.durabilityLevel
}
deadline, duraTimeout := transactionsMutationTimeouts(c.keyValueTimeout, req.DurabilityLevel)
_, err = agent.MutateIn(MutateInOptions{
Key: req.AtrID,
ScopeName: req.AtrScopeName,
CollectionName: req.AtrCollectionName,
Ops: specs,
Deadline: deadline,
DurabilityLevel: transactionsDurabilityLevelToMemd(req.DurabilityLevel),
DurabilityLevelTimeout: duraTimeout,
User: oboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
c.updateResourceUnitsError(err)
if errors.Is(err, ErrPathNotFound) {
cb(nil)
return
}
logDebugf("Failed to cleanup ATR for request: %s, err: %v", req.String(), err)
cb(err)
return
}
c.updateResourceUnits(result.Internal.ResourceUnits)
cb(nil)
})
if err != nil {
cb(err)
return
}
})
}
func (c *stdTransactionsCleaner) cleanupDocs(req *TransactionsCleanupRequest, cb func(error)) {
var memdDuraLevel memd.DurabilityLevel
if req.DurabilityLevel > TransactionDurabilityLevelUnknown {
// We want to ensure that we don't panic here, if the durability level is unknown then we'll just not set
// a durability level.
memdDuraLevel = transactionsDurabilityLevelToMemd(req.DurabilityLevel)
}
deadline, duraTimeout := transactionsMutationTimeouts(c.keyValueTimeout, req.DurabilityLevel)
switch req.State {
case TransactionAttemptStateCommitted:
waitCh := make(chan error, 1)
c.commitInsRepDocs(req.AttemptID, req.Inserts, deadline, memdDuraLevel, duraTimeout, func(err error) {
waitCh <- err
})
err := <-waitCh
if err != nil {
cb(err)
return
}
waitCh = make(chan error, 1)
c.commitInsRepDocs(req.AttemptID, req.Replaces, deadline, memdDuraLevel, duraTimeout, func(err error) {
waitCh <- err
})
err = <-waitCh
if err != nil {
cb(err)
return
}
waitCh = make(chan error, 1)
c.commitRemDocs(req.AttemptID, req.Removes, deadline, memdDuraLevel, duraTimeout, func(err error) {
waitCh <- err
})
err = <-waitCh
if err != nil {
cb(err)
return
}
cb(nil)
case TransactionAttemptStateAborted:
waitCh := make(chan error, 1)
c.rollbackInsDocs(req.AttemptID, req.Inserts, deadline, memdDuraLevel, duraTimeout, func(err error) {
waitCh <- err
})
err := <-waitCh
if err != nil {
cb(err)
return
}
waitCh = make(chan error, 1)
c.rollbackRepRemDocs(req.AttemptID, req.Replaces, deadline, memdDuraLevel, duraTimeout, func(err error) {
waitCh <- err
})
err = <-waitCh
if err != nil {
cb(err)
return
}
waitCh = make(chan error, 1)
c.rollbackRepRemDocs(req.AttemptID, req.Removes, deadline, memdDuraLevel, duraTimeout, func(err error) {
waitCh <- err
})
err = <-waitCh
if err != nil {
cb(err)
return
}
cb(nil)
case TransactionAttemptStatePending:
cb(nil)
case TransactionAttemptStateCompleted:
cb(nil)
case TransactionAttemptStateRolledBack:
cb(nil)
case TransactionAttemptStateNothingWritten:
cb(nil)
default:
cb(nil)
}
}
func (c *stdTransactionsCleaner) rollbackRepRemDocs(attemptID string, docs []TransactionsDocRecord, deadline time.Time, durability memd.DurabilityLevel,
duraTimeout time.Duration, cb func(err error)) {
for _, doc := range docs {
waitCh := make(chan error, 1)
agent, oboUser, err := c.bucketAgentProvider(doc.BucketName)
if err != nil {
cb(err)
return
}
c.perDoc(false, attemptID, doc, agent, oboUser, func(getRes *transactionGetDoc, err error) {
if err != nil {
waitCh <- err
return
}
if getRes == nil {
// This violates implicit contract idioms but needs must.
waitCh <- nil
return
}
c.hooks.BeforeRemoveLinks(doc.ID, func(err error) {
if err != nil {
waitCh <- err
return
}
_, err = agent.MutateIn(MutateInOptions{
Key: doc.ID,
ScopeName: doc.ScopeName,
CollectionName: doc.CollectionName,
Cas: getRes.Cas,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDelete,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
},
Flags: memd.SubdocDocFlagAccessDeleted,
Deadline: deadline,
DurabilityLevel: durability,
DurabilityLevelTimeout: duraTimeout,
User: oboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
c.updateResourceUnitsError(err)
logDebugf("Failed to rollback for bucket: %s, collection: %s, scope: %s, id: %s, err: %v",
doc.BucketName, doc.CollectionName, doc.ScopeName, doc.ID, err)
waitCh <- err
return
}
c.updateResourceUnits(result.Internal.ResourceUnits)
waitCh <- nil
})
if err != nil {
waitCh <- err
return
}
})
})
err = <-waitCh
if err != nil {
cb(err)
return
}
}
cb(nil)
}
func (c *stdTransactionsCleaner) rollbackInsDocs(attemptID string, docs []TransactionsDocRecord, deadline time.Time, durability memd.DurabilityLevel,
duraTimeout time.Duration, cb func(err error)) {
for _, doc := range docs {
waitCh := make(chan error, 1)
agent, oboUser, err := c.bucketAgentProvider(doc.BucketName)
if err != nil {
cb(err)
return
}
c.perDoc(false, attemptID, doc, agent, oboUser, func(getRes *transactionGetDoc, err error) {
if err != nil {
waitCh <- err
return
}
if getRes == nil {
// This violates implicit contract idioms but needs must.
waitCh <- nil
return
}
c.hooks.BeforeRemoveDoc(doc.ID, func(err error) {
if err != nil {
waitCh <- err
return
}
if getRes.Deleted {
_, err := agent.MutateIn(MutateInOptions{
Key: doc.ID,
ScopeName: doc.ScopeName,
CollectionName: doc.CollectionName,
Cas: getRes.Cas,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDelete,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
},
Flags: memd.SubdocDocFlagAccessDeleted,
Deadline: deadline,
DurabilityLevel: durability,
DurabilityLevelTimeout: duraTimeout,
User: oboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
c.updateResourceUnitsError(err)
logDebugf("Failed to rollback for bucket: %s, collection: %s, scope: %s, id: %s, err: %v",
doc.BucketName, doc.CollectionName, doc.ScopeName, doc.ID, err)
waitCh <- err
return
}
c.updateResourceUnits(result.Internal.ResourceUnits)
waitCh <- nil
})
if err != nil {
waitCh <- err
return
}
} else {
_, err := agent.Delete(DeleteOptions{
Key: doc.ID,
ScopeName: doc.ScopeName,
CollectionName: doc.CollectionName,
Cas: getRes.Cas,
Deadline: deadline,
DurabilityLevel: durability,
DurabilityLevelTimeout: duraTimeout,
User: oboUser,
}, func(result *DeleteResult, err error) {
if err != nil {
c.updateResourceUnitsError(err)
logDebugf("Failed to rollback for bucket: %s, collection: %s, scope: %s, id: %s, err: %v",
doc.BucketName, doc.CollectionName, doc.ScopeName, doc.ID, err)
waitCh <- err
return
}
c.updateResourceUnits(result.Internal.ResourceUnits)
waitCh <- nil
})
if err != nil {
waitCh <- err
return
}
}
})
})
err = <-waitCh
if err != nil {
cb(err)
return
}
}
cb(nil)
}
func (c *stdTransactionsCleaner) commitRemDocs(attemptID string, docs []TransactionsDocRecord, deadline time.Time, durability memd.DurabilityLevel,
duraTimeout time.Duration, cb func(err error)) {
for _, doc := range docs {
waitCh := make(chan error, 1)
agent, oboUser, err := c.bucketAgentProvider(doc.BucketName)
if err != nil {
cb(err)
return
}
c.perDoc(true, attemptID, doc, agent, oboUser, func(getRes *transactionGetDoc, err error) {
if err != nil {
waitCh <- err
return
}
if getRes == nil {
// This violates implicit contract idioms but needs must.
waitCh <- nil
return
}
c.hooks.BeforeRemoveDocStagedForRemoval(doc.ID, func(err error) {
if err != nil {
waitCh <- err
return
}
if getRes.TxnMeta.Operation.Type != jsonMutationRemove {
waitCh <- nil
return
}
_, err = agent.Delete(DeleteOptions{
Key: doc.ID,
ScopeName: doc.ScopeName,
CollectionName: doc.CollectionName,
Cas: getRes.Cas,
Deadline: deadline,
DurabilityLevel: durability,
DurabilityLevelTimeout: duraTimeout,
User: oboUser,
}, func(result *DeleteResult, err error) {
if err != nil {
c.updateResourceUnitsError(err)
logDebugf("Failed to commit for bucket: %s, collection: %s, scope: %s, id: %s, err: %v",
doc.BucketName, doc.CollectionName, doc.ScopeName, doc.ID, err)
waitCh <- err
return
}
c.updateResourceUnits(result.Internal.ResourceUnits)
waitCh <- nil
})
if err != nil {
waitCh <- err
return
}
})
})
err = <-waitCh
if err != nil {
cb(err)
return
}
}
cb(nil)
}
func (c *stdTransactionsCleaner) commitInsRepDocs(attemptID string, docs []TransactionsDocRecord, deadline time.Time, durability memd.DurabilityLevel,
duraTimeout time.Duration, cb func(err error)) {
for _, doc := range docs {
waitCh := make(chan error, 1)
agent, oboUser, err := c.bucketAgentProvider(doc.BucketName)
if err != nil {
cb(err)
return
}
c.perDoc(true, attemptID, doc, agent, oboUser, func(getRes *transactionGetDoc, err error) {
if err != nil {
waitCh <- err
return
}
if getRes == nil {
// This violates implicit contract idioms but needs must.
waitCh <- nil
return
}
c.hooks.BeforeCommitDoc(doc.ID, func(err error) {
if err != nil {
waitCh <- err
return
}
if getRes.Deleted {
_, err := agent.Set(SetOptions{
Value: getRes.Body,
Key: doc.ID,
ScopeName: doc.ScopeName,
CollectionName: doc.CollectionName,
Deadline: deadline,
DurabilityLevel: durability,
DurabilityLevelTimeout: duraTimeout,
User: oboUser,
}, func(result *StoreResult, err error) {
if err != nil {
c.updateResourceUnitsError(err)
logDebugf("Failed to commit for bucket: %s, collection: %s, scope: %s, id: %s, err: %v",
doc.BucketName, doc.CollectionName, doc.ScopeName, doc.ID, err)
waitCh <- err
return
}
c.updateResourceUnits(result.Internal.ResourceUnits)
waitCh <- nil
})
if err != nil {
waitCh <- err
return
}
} else {
_, err := agent.MutateIn(MutateInOptions{
Key: doc.ID,
ScopeName: doc.ScopeName,
CollectionName: doc.CollectionName,
Cas: getRes.Cas,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDelete,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
{
Op: memd.SubDocOpSetDoc,
Path: "",
Value: getRes.Body,
},
},
Deadline: deadline,
DurabilityLevel: durability,
DurabilityLevelTimeout: duraTimeout,
User: oboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
c.updateResourceUnitsError(err)
logDebugf("Failed to commit for bucket: %s, collection: %s, scope: %s, id: %s, err: %v",
doc.BucketName, doc.CollectionName, doc.ScopeName, doc.ID, err)
waitCh <- err
return
}
c.updateResourceUnits(result.Internal.ResourceUnits)
waitCh <- nil
})
if err != nil {
waitCh <- err
return
}
}
})
})
err = <-waitCh
if err != nil {
cb(err)
return
}
}
cb(nil)
}
func (c *stdTransactionsCleaner) perDoc(crc32MatchStaging bool, attemptID string, dr TransactionsDocRecord, agent *Agent, oboUser string,
cb func(getRes *transactionGetDoc, err error)) {
c.hooks.BeforeDocGet(dr.ID, func(err error) {
if err != nil {
cb(nil, err)
return
}
var deadline time.Time
if c.keyValueTimeout > 0 {
deadline = time.Now().Add(c.keyValueTimeout)
}
_, err = agent.LookupIn(LookupInOptions{
ScopeName: dr.ScopeName,
CollectionName: dr.CollectionName,
Key: dr.ID,
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "$document",
Flags: memd.SubdocFlagXattrPath,
},
{
Op: memd.SubDocOpGet,
Path: "txn",
Flags: memd.SubdocFlagXattrPath,
},
},
Deadline: deadline,
Flags: memd.SubdocDocFlagAccessDeleted,
User: oboUser,
}, func(result *LookupInResult, err error) {
if err != nil {
c.updateResourceUnitsError(err)
if errors.Is(err, ErrDocumentNotFound) {
// We can consider this success.
cb(nil, nil)
return
}
logDebugf("Failed to lookup doc for bucket: %s, collection: %s, scope: %s, id: %s, err: %v",
dr.BucketName, dr.CollectionName, dr.ScopeName, dr.ID, err)
cb(nil, err)
return
}
c.updateResourceUnits(result.Internal.ResourceUnits)
if result.Ops[0].Err != nil {
// This is not so good.
cb(nil, result.Ops[0].Err)
return
}
if result.Ops[1].Err != nil {
// Txn probably committed so this is success.
cb(nil, nil)
return
}
var txnMetaVal *jsonTxnXattr
if err := json.Unmarshal(result.Ops[1].Value, &txnMetaVal); err != nil {
cb(nil, err)
return
}
if attemptID != txnMetaVal.ID.Attempt {
// Document involved in another txn, was probably committed, this is success.
cb(nil, nil)
return
}
var meta *transactionDocMeta
if err := json.Unmarshal(result.Ops[0].Value, &meta); err != nil {
cb(nil, err)
return
}
if crc32MatchStaging {
if meta.CRC32 != txnMetaVal.Operation.CRC32 {
// This document is a part of this txn but its body has changed, we'll continue as success.
cb(nil, nil)
return
}
}
cb(&transactionGetDoc{
Body: txnMetaVal.Operation.Staged,
DocMeta: meta,
Cas: result.Cas,
Deleted: result.Internal.IsDeleted,
TxnMeta: txnMetaVal,
}, nil)
})
if err != nil {
cb(nil, err)
}
})
}
gocbcore-10.2.3/transactions_config.go 0000664 0000000 0000000 00000022777 14417540156 0017772 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"errors"
"fmt"
"time"
)
// TransactionDurabilityLevel specifies the durability level to use for a mutation.
type TransactionDurabilityLevel int
const (
// TransactionDurabilityLevelUnknown indicates to use the default level.
TransactionDurabilityLevelUnknown = TransactionDurabilityLevel(0)
// TransactionDurabilityLevelNone indicates that no durability is needed.
TransactionDurabilityLevelNone = TransactionDurabilityLevel(1)
// TransactionDurabilityLevelMajority indicates the operation must be replicated to the majority.
TransactionDurabilityLevelMajority = TransactionDurabilityLevel(2)
// TransactionDurabilityLevelMajorityAndPersistToActive indicates the operation must be replicated
// to the majority and persisted to the active server.
TransactionDurabilityLevelMajorityAndPersistToActive = TransactionDurabilityLevel(3)
// TransactionDurabilityLevelPersistToMajority indicates the operation must be persisted to the active server.
TransactionDurabilityLevelPersistToMajority = TransactionDurabilityLevel(4)
)
func transactionDurabilityLevelToString(level TransactionDurabilityLevel) string {
switch level {
case TransactionDurabilityLevelUnknown:
return "UNSET"
case TransactionDurabilityLevelNone:
return "NONE"
case TransactionDurabilityLevelMajority:
return "MAJORITY"
case TransactionDurabilityLevelMajorityAndPersistToActive:
return "MAJORITY_AND_PERSIST_TO_ACTIVE"
case TransactionDurabilityLevelPersistToMajority:
return "PERSIST_TO_MAJORITY"
}
return ""
}
func transactionDurabilityLevelFromString(level string) (TransactionDurabilityLevel, error) {
switch level {
case "UNSET":
return TransactionDurabilityLevelUnknown, nil
case "NONE":
return TransactionDurabilityLevelNone, nil
case "MAJORITY":
return TransactionDurabilityLevelMajority, nil
case "MAJORITY_AND_PERSIST_TO_ACTIVE":
return TransactionDurabilityLevelMajorityAndPersistToActive, nil
case "PERSIST_TO_MAJORITY":
return TransactionDurabilityLevelPersistToMajority, nil
}
return TransactionDurabilityLevelUnknown, errors.New("invalid durability level string")
}
// TransactionATRLocation specifies a specific location where ATR entries should be
// placed when performing transactions.
type TransactionATRLocation struct {
Agent *Agent
OboUser string
ScopeName string
CollectionName string
}
func (tlal TransactionATRLocation) build() string {
if tlal.Agent == nil {
return ""
}
scope := tlal.ScopeName
if scope == "" {
scope = "_default"
}
collection := tlal.CollectionName
if collection == "" {
collection = "_default"
}
return tlal.Agent.BucketName() + "." + scope + "." + collection
}
func (tlal TransactionATRLocation) String() string {
if isLogRedactionLevelFull() || isLogRedactionLevelPartial() {
return redactMetaData(tlal.build())
}
return tlal.build()
}
func (tlal TransactionATRLocation) redacted() interface{} {
return redactMetaData(tlal.build())
}
// TransactionLostATRLocation specifies a specific location where lost transactions should
// attempt cleanup.
type TransactionLostATRLocation struct {
BucketName string
ScopeName string
CollectionName string
}
func (tlal TransactionLostATRLocation) build() string {
if tlal.BucketName == "" {
return ""
}
scope := tlal.ScopeName
if scope == "" {
scope = "_default"
}
collection := tlal.CollectionName
if collection == "" {
collection = "_default"
}
return tlal.BucketName + "." + scope + "." + collection
}
func (tlal TransactionLostATRLocation) String() string {
if isLogRedactionLevelFull() || isLogRedactionLevelPartial() {
return redactMetaData(tlal.build())
}
return tlal.build()
}
func (tlal TransactionLostATRLocation) redacted() interface{} {
return redactMetaData(tlal.build())
}
// TransactionsBucketAgentProviderFn is a function used to provide an agent for
// a particular bucket by name.
type TransactionsBucketAgentProviderFn func(bucketName string) (*Agent, string, error)
// TransactionsLostCleanupATRLocationProviderFn is a function used to provide a list of ATRLocations for
// lost transactions cleanup.
type TransactionsLostCleanupATRLocationProviderFn func() ([]TransactionLostATRLocation, error)
// TransactionsConfig specifies various tunable options related to transactions.
type TransactionsConfig struct {
// CustomATRLocation specifies a specific location to place meta-data.
CustomATRLocation TransactionATRLocation
// ExpirationTime sets the maximum time that transactions created
// by this TransactionsManager object can run for, before expiring.
ExpirationTime time.Duration
// DurabilityLevel specifies the durability level that should be used
// for all write operations performed by this TransactionsManager object.
DurabilityLevel TransactionDurabilityLevel
// KeyValueTimeout specifies the default timeout used for all KV writes.
KeyValueTimeout time.Duration
// CleanupWindow specifies how often to the cleanup process runs
// attempting to garbage collection transactions that have failed but
// were not cleaned up by the previous client.
CleanupWindow time.Duration
// CleanupClientAttempts controls where any transaction attempts made
// by this client are automatically removed.
CleanupClientAttempts bool
// CleanupLostAttempts controls where a background process is created
// to cleanup any ‘lost’ transaction attempts.
CleanupLostAttempts bool
// CleanupQueueSize controls the maximum queue size for the cleanup thread.
CleanupQueueSize uint32
// BucketAgentProvider provides a function which returns an agent for
// a particular bucket by name.
BucketAgentProvider TransactionsBucketAgentProviderFn
// LostCleanupATRLocationProvider provides a function which returns a list of LostATRLocations
// for use in lost transaction cleanup.
LostCleanupATRLocationProvider TransactionsLostCleanupATRLocationProviderFn
// CleanupWatchATRs is *NOT* used within the codebase, it is *only* here to provide API level backward
// compatibility.
// This should *never* be used.
CleanupWatchATRs bool
// Internal specifies a set of options for internal use.
// Internal: This should never be used and is not supported.
Internal struct {
Hooks TransactionHooks
CleanUpHooks TransactionCleanUpHooks
ClientRecordHooks TransactionClientRecordHooks
EnableNonFatalGets bool
EnableParallelUnstaging bool
EnableExplicitATRs bool
EnableMutationCaching bool
NumATRs int
}
}
func (config *TransactionsConfig) String() string {
if config == nil {
return ""
}
return fmt.Sprintf("CustomATRLocation:%s ExpirationTime:%s DurabilityLevel:%s KeyValueTimeout:%s CleanupWindow:%s "+
"CleanupClientAttempts:%t CleanupLostAttempts:%t CleanupQueueSize:%d BucketAgentProvider:%p LostCleanupATRLocationProvider:%p "+
"Internal:{EnableNonFatalGets:%t EnableParallelUnstaging:%t "+"EnableExplicitATRs:%t EnableMutationCaching:%t NumATRs:%d}",
config.CustomATRLocation, config.ExpirationTime, transactionDurabilityLevelToString(config.DurabilityLevel),
config.KeyValueTimeout, config.CleanupWindow, config.CleanupClientAttempts, config.CleanupLostAttempts, config.CleanupQueueSize,
config.BucketAgentProvider, config.LostCleanupATRLocationProvider, config.Internal.EnableNonFatalGets,
config.Internal.EnableParallelUnstaging, config.Internal.EnableExplicitATRs, config.Internal.EnableMutationCaching,
config.Internal.NumATRs)
}
// TransactionOptions specifies options which can be overridden on a per transaction basis.
type TransactionOptions struct {
// CustomATRLocation specifies a specific location to place meta-data.
CustomATRLocation TransactionATRLocation
// ExpirationTime sets the maximum time that this transaction will
// run for, before expiring.
ExpirationTime time.Duration
// DurabilityLevel specifies the durability level that should be used
// for all write operations performed by this transaction.
DurabilityLevel TransactionDurabilityLevel
// KeyValueTimeout specifies the timeout used for all KV writes.
KeyValueTimeout time.Duration
// BucketAgentProvider provides a function which returns an agent for
// a particular bucket by name.
BucketAgentProvider TransactionsBucketAgentProviderFn
// TransactionLogger is the logger to use with this transaction.
// Uncommitted: This API may change in the future.
TransactionLogger TransactionLogger
// Internal specifies a set of options for internal use.
// Internal: This should never be used and is not supported.
Internal struct {
Hooks TransactionHooks
ResourceUnitCallback func(result *ResourceUnitResult)
}
}
func (opts *TransactionOptions) String() string {
if opts == nil {
return ""
}
return fmt.Sprintf("CustomATRLocation:%s ExpirationTime:%s DurabilityLevel:%s KeyValueTimeout:%s "+
"BucketAgentProvider:%p TransactionLogger:%p ",
opts.CustomATRLocation, opts.ExpirationTime, transactionDurabilityLevelToString(opts.DurabilityLevel),
opts.KeyValueTimeout, opts.BucketAgentProvider, opts.TransactionLogger)
}
gocbcore-10.2.3/transactions_constants.go 0000664 0000000 0000000 00000014534 14417540156 0020531 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import "fmt"
// TransactionAttemptState represents the current State of a transaction
type TransactionAttemptState int
const (
// TransactionAttemptStateNothingWritten indicates that nothing has been written yet.
TransactionAttemptStateNothingWritten = TransactionAttemptState(1)
// TransactionAttemptStatePending indicates that the transaction ATR has been written and
// the transaction is currently pending.
TransactionAttemptStatePending = TransactionAttemptState(2)
// TransactionAttemptStateCommitting indicates that the transaction is now trying to become
// committed, if we stay in this state, it implies ambiguity.
TransactionAttemptStateCommitting = TransactionAttemptState(3)
// TransactionAttemptStateCommitted indicates that the transaction is now logically committed
// but the unstaging of documents is still underway.
TransactionAttemptStateCommitted = TransactionAttemptState(4)
// TransactionAttemptStateCompleted indicates that the transaction has been fully completed
// and no longer has work to perform.
TransactionAttemptStateCompleted = TransactionAttemptState(5)
// TransactionAttemptStateAborted indicates that the transaction was aborted.
TransactionAttemptStateAborted = TransactionAttemptState(6)
// TransactionAttemptStateRolledBack indicates that the transaction was not committed and instead
// was rolled back in its entirety.
TransactionAttemptStateRolledBack = TransactionAttemptState(7)
)
func (state TransactionAttemptState) String() string {
switch state {
case TransactionAttemptStateNothingWritten:
return "nothing_written"
case TransactionAttemptStatePending:
return "pending"
case TransactionAttemptStateCommitting:
return "committing"
case TransactionAttemptStateCommitted:
return "committed"
case TransactionAttemptStateCompleted:
return "completed"
case TransactionAttemptStateAborted:
return "aborted"
case TransactionAttemptStateRolledBack:
return "rolled_back"
default:
return "unknown"
}
}
// TransactionErrorReason is the reason why a transaction should be failed.
// Internal: This should never be used and is not supported.
type TransactionErrorReason uint8
// NOTE: The errors within this section are critically ordered, as the order of
// precedence used when merging errors together is based on this.
const (
// TransactionErrorReasonSuccess indicates the transaction succeeded and did not fail.
TransactionErrorReasonSuccess TransactionErrorReason = iota
// TransactionErrorReasonTransactionFailed indicates the transaction should be failed because it failed.
TransactionErrorReasonTransactionFailed
// TransactionErrorReasonTransactionExpired indicates the transaction should be failed because it expired.
TransactionErrorReasonTransactionExpired
// TransactionErrorReasonTransactionCommitAmbiguous indicates the transaction should be failed and the commit was ambiguous.
TransactionErrorReasonTransactionCommitAmbiguous
// TransactionErrorReasonTransactionFailedPostCommit indicates the transaction should be failed because it failed post commit.
TransactionErrorReasonTransactionFailedPostCommit
)
func (reason TransactionErrorReason) String() string {
switch reason {
case TransactionErrorReasonTransactionFailed:
return "failed"
case TransactionErrorReasonTransactionExpired:
return "expired"
case TransactionErrorReasonTransactionCommitAmbiguous:
return "commit_ambiguous"
case TransactionErrorReasonTransactionFailedPostCommit:
return "failed_post_commit"
default:
return fmt.Sprintf("unknown:%d", reason)
}
}
// TransactionErrorClass describes the reason that a transaction error occurred.
// Internal: This should never be used and is not supported.
type TransactionErrorClass uint8
const (
// TransactionErrorClassFailOther indicates an error occurred because it did not fit into any other reason.
TransactionErrorClassFailOther TransactionErrorClass = iota
// TransactionErrorClassFailTransient indicates an error occurred because of a transient reason.
TransactionErrorClassFailTransient
// TransactionErrorClassFailDocNotFound indicates an error occurred because of a document not found.
TransactionErrorClassFailDocNotFound
// TransactionErrorClassFailDocAlreadyExists indicates an error occurred because a document already exists.
TransactionErrorClassFailDocAlreadyExists
// TransactionErrorClassFailPathNotFound indicates an error occurred because a path was not found.
TransactionErrorClassFailPathNotFound
// TransactionErrorClassFailPathAlreadyExists indicates an error occurred because a path already exists.
TransactionErrorClassFailPathAlreadyExists
// TransactionErrorClassFailWriteWriteConflict indicates an error occurred because of a write write conflict.
TransactionErrorClassFailWriteWriteConflict
// TransactionErrorClassFailCasMismatch indicates an error occurred because of a cas mismatch.
TransactionErrorClassFailCasMismatch
// TransactionErrorClassFailHard indicates an error occurred because of a hard error.
TransactionErrorClassFailHard
// TransactionErrorClassFailAmbiguous indicates an error occurred leaving the transaction in an ambiguous way.
TransactionErrorClassFailAmbiguous
// TransactionErrorClassFailExpiry indicates an error occurred because the transaction expired.
TransactionErrorClassFailExpiry
// TransactionErrorClassFailOutOfSpace indicates an error occurred because the ATR is full.
TransactionErrorClassFailOutOfSpace
)
const (
transactionStateBitShouldNotCommit = 1 << 0
transactionStateBitShouldNotRollback = 1 << 1
transactionStateBitShouldNotRetry = 1 << 2
transactionStateBitHasExpired = 1 << 3
transactionStateBitPreExpiryAutoRollback = 1 << 4
)
const (
transactionStateBitsMaskFinalError = 0b1110000
transactionStateBitsMaskBits = 0b0001111
transactionStateBitsPositionFinalError = 4
)
gocbcore-10.2.3/transactions_forwardcompatibility.go 0000664 0000000 0000000 00000016561 14417540156 0022755 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"fmt"
"strconv"
"strings"
"time"
)
// TransactionsProtocolVersion returns the protocol version that this library supports.
func TransactionsProtocolVersion() string {
return "2.1"
}
// TransactionsProtocolExtensions returns a list strings representing the various features
// that this specific version of the library supports within its protocol version.
func TransactionsProtocolExtensions() []string {
return []string{
"EXT_TRANSACTION_ID",
"EXT_MEMORY_OPT_UNSTAGING",
"EXT_BINARY_METADATA",
"EXT_CUSTOM_METADATA_COLLECTION",
"EXT_STORE_DURABILITY",
"EXT_REMOVE_COMPLETED",
"EXT_ALL_KV_COMBINATIONS",
"EXT_UNKNOWN_ATR_STATES",
"BF_CBD_3787",
"BF_CBD_3705",
"BF_CBD_3838",
"BF_CBD_3791",
"BF_CBD_3794",
"EXT_QUERY",
"EXT_SDK_INTEGRATION",
"EXT_SINGLE_QUERY",
"EXT_INSERT_EXISTING",
"EXT_QUERY_CONTEXT",
}
}
type forwardCompatBehaviour string
// nolint: deadcode,varcheck
const (
forwardCompatBehaviourRetry forwardCompatBehaviour = "r"
forwardCompatBehaviourFail forwardCompatBehaviour = "f"
)
type forwardCompatExtension string
// nolint: deadcode,varcheck
const (
forwardCompatExtensionTransactionID forwardCompatExtension = "TI"
forwardCompatExtensionDeferredCommit forwardCompatExtension = "DC"
forwardCompatExtensionTimeOptUnstaging forwardCompatExtension = "TO"
forwardCompatExtensionMemoryOptUnstaging forwardCompatExtension = "MO"
forwardCompatExtensionCustomMetadataCollection forwardCompatExtension = "CM"
forwardCompatExtensionBinaryMetadata forwardCompatExtension = "BM"
forwardCompatExtensionQuery forwardCompatExtension = "QU"
forwardCompatExtensionStoreDurability forwardCompatExtension = "SD"
forwardCompatExtensionRemoveCompleted forwardCompatExtension = "RC"
forwardCompatExtensionAllKvCombinations forwardCompatExtension = "CO"
forwardCompatExtensionUnknownATRStates forwardCompatExtension = "UA"
forwardCompatExtensionBFCBD3787 forwardCompatExtension = "BF3787"
forwardCompatExtensionBFCBD3705 forwardCompatExtension = "BF3705"
forwardCompatExtensionBFCBD3838 forwardCompatExtension = "BF3838"
forwardCompatExtensionBFCBD3791 forwardCompatExtension = "BF3791"
forwardCompatExtensionBFCBD3794 forwardCompatExtension = "BF3794"
forwardCompatExtensionSDKIntegration forwardCompatExtension = "SI"
forwardCompatExtensionSingleQuery forwardCompatExtension = "SQ"
forwardCompatExtensionInsertExisting forwardCompatExtension = "IX"
forwardCompatExtensionQueryContext forwardCompatExtension = "QC"
)
type forwardCompatStage string
// nolint: deadcode,varcheck
const (
forwardCompatStageWWCReadingATR forwardCompatStage = "WW_R"
forwardCompatStageWWCReplacing forwardCompatStage = "WW_RP"
forwardCompatStageWWCRemoving forwardCompatStage = "WW_RM"
forwardCompatStageWWCInserting forwardCompatStage = "WW_I"
forwardCompatStageWWCInsertingGet forwardCompatStage = "WW_IG"
forwardCompatStageGets forwardCompatStage = "G"
forwardCompatStageGetsReadingATR forwardCompatStage = "G_A"
forwardCompatStageGetsCleanupEntry forwardCompatStage = "CL_E"
)
const (
protocolMajor = 2
protocolMinor = 0
)
// TransactionForwardCompatibilityEntry represents a forward compatibility entry.
// Internal: This should never be used and is not supported.
type TransactionForwardCompatibilityEntry struct {
ProtocolVersion string `json:"p,omitempty"`
ProtocolExtension string `json:"e,omitempty"`
Behaviour string `json:"b,omitempty"`
RetryInterval int `json:"ra,omitempty"`
}
var supportedforwardCompatExtensions = []forwardCompatExtension{
forwardCompatExtensionTransactionID,
forwardCompatExtensionMemoryOptUnstaging,
forwardCompatExtensionCustomMetadataCollection,
forwardCompatExtensionBinaryMetadata,
forwardCompatExtensionQuery,
forwardCompatExtensionStoreDurability,
forwardCompatExtensionRemoveCompleted,
forwardCompatExtensionAllKvCombinations,
forwardCompatExtensionUnknownATRStates,
forwardCompatExtensionBFCBD3787,
forwardCompatExtensionBFCBD3705,
forwardCompatExtensionBFCBD3838,
forwardCompatExtensionBFCBD3791,
forwardCompatExtensionBFCBD3794,
forwardCompatExtensionSDKIntegration,
forwardCompatExtensionSingleQuery,
forwardCompatExtensionInsertExisting,
forwardCompatExtensionQueryContext,
}
func jsonForwardCompatToForwardCompat(fc map[string][]jsonForwardCompatibilityEntry) map[string][]TransactionForwardCompatibilityEntry {
if fc == nil {
return nil
}
forwardCompat := make(map[string][]TransactionForwardCompatibilityEntry)
for k, entries := range fc {
if _, ok := forwardCompat[k]; !ok {
forwardCompat[k] = make([]TransactionForwardCompatibilityEntry, len(entries))
}
for i, entry := range entries {
forwardCompat[k][i] = TransactionForwardCompatibilityEntry(entry)
}
}
return forwardCompat
}
func checkForwardCompatProtocol(protocolVersion string) (bool, error) {
if protocolVersion == "" {
return false, nil
}
protocol := strings.Split(protocolVersion, ".")
if len(protocol) != 2 {
return false, fmt.Errorf("invalid protocol string: %s", protocolVersion)
}
major, err := strconv.Atoi(protocol[0])
if err != nil {
return false, wrapError(err, fmt.Sprintf("invalid protocol string: %s", protocolVersion))
}
if protocolMajor < major {
return false, nil
}
if protocolMajor == major {
minor, err := strconv.Atoi(protocol[1])
if err != nil {
return false, wrapError(err, fmt.Sprintf("invalid protocol string: %s", protocolVersion))
}
if protocolMinor < minor {
return false, nil
}
}
return true, nil
}
func checkForwardCompatExtension(extension string) bool {
if extension == "" {
return false
}
for _, supported := range supportedforwardCompatExtensions {
if string(supported) == extension {
return true
}
}
return false
}
func checkForwardCompatability(
stage forwardCompatStage,
fc map[string][]TransactionForwardCompatibilityEntry,
) (isCompatOut bool, shouldRetryOut bool, retryWaitOut time.Duration, errOut error) {
if len(fc) == 0 {
return true, false, 0, nil
}
if checks, ok := fc[string(stage)]; ok {
for _, c := range checks {
protocolOk, err := checkForwardCompatProtocol(c.ProtocolVersion)
if err != nil {
return false, false, 0, err
}
if protocolOk {
continue
}
if extensionOk := checkForwardCompatExtension(c.ProtocolExtension); extensionOk {
continue
}
// If we get here then neither protocol or extension are ok.
switch forwardCompatBehaviour(c.Behaviour) {
case forwardCompatBehaviourRetry:
retryWait := time.Duration(c.RetryInterval) * time.Millisecond
return false, true, retryWait, nil
default:
return false, false, 0, nil
}
}
}
return true, false, 0, nil
}
gocbcore-10.2.3/transactions_helpers_test.go 0000664 0000000 0000000 00000005356 14417540156 0021220 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
func testBlkGet(txn *Transaction, opts TransactionGetOptions) (resOut *TransactionGetResult, errOut error) {
waitCh := make(chan struct{}, 1)
err := txn.Get(opts, func(res *TransactionGetResult, err error) {
resOut = res
errOut = err
waitCh <- struct{}{}
})
if err != nil {
resOut = nil
errOut = err
return
}
<-waitCh
return
}
func testBlkInsert(txn *Transaction, opts TransactionInsertOptions) (resOut *TransactionGetResult, errOut error) {
waitCh := make(chan struct{}, 1)
err := txn.Insert(opts, func(res *TransactionGetResult, err error) {
resOut = res
errOut = err
waitCh <- struct{}{}
})
if err != nil {
resOut = nil
errOut = err
return
}
<-waitCh
return
}
func testBlkReplace(txn *Transaction, opts TransactionReplaceOptions) (resOut *TransactionGetResult, errOut error) {
waitCh := make(chan struct{}, 1)
err := txn.Replace(opts, func(res *TransactionGetResult, err error) {
resOut = res
errOut = err
waitCh <- struct{}{}
})
if err != nil {
resOut = nil
errOut = err
return
}
<-waitCh
return
}
func testBlkRemove(txn *Transaction, opts TransactionRemoveOptions) (resOut *TransactionGetResult, errOut error) {
waitCh := make(chan struct{}, 1)
err := txn.Remove(opts, func(res *TransactionGetResult, err error) {
resOut = res
errOut = err
waitCh <- struct{}{}
})
if err != nil {
resOut = nil
errOut = err
return
}
<-waitCh
return
}
func testBlkCommit(txn *Transaction) (errOut error) {
waitCh := make(chan struct{}, 1)
err := txn.Commit(func(err error) {
errOut = err
waitCh <- struct{}{}
})
if err != nil {
errOut = err
return
}
<-waitCh
return
}
func testBlkRollback(txn *Transaction) (errOut error) {
waitCh := make(chan struct{}, 1)
err := txn.Rollback(func(err error) {
errOut = err
waitCh <- struct{}{}
})
if err != nil {
errOut = err
return
}
<-waitCh
return
}
func testBlkSerialize(txn *Transaction) (txnBytesOut []byte, errOut error) {
waitCh := make(chan struct{}, 1)
err := txn.SerializeAttempt(func(txnBytes []byte, err error) {
txnBytesOut = txnBytes
errOut = err
waitCh <- struct{}{}
})
if err != nil {
errOut = err
return
}
<-waitCh
return
}
gocbcore-10.2.3/transactions_hooks.go 0000664 0000000 0000000 00000032162 14417540156 0017635 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
// TransactionHooks provides a number of internal hooks used for testing.
// Internal: This should never be used and is not supported.
type TransactionHooks interface {
BeforeATRCommit(func(err error))
AfterATRCommit(func(err error))
BeforeDocCommitted(docID []byte, cb func(err error))
BeforeRemovingDocDuringStagedInsert(docID []byte, cb func(err error))
BeforeRollbackDeleteInserted(docID []byte, cb func(err error))
AfterDocCommittedBeforeSavingCAS(docID []byte, cb func(err error))
AfterDocCommitted(docID []byte, cb func(err error))
BeforeStagedInsert(docID []byte, cb func(err error))
BeforeStagedRemove(docID []byte, cb func(err error))
BeforeStagedReplace(docID []byte, cb func(err error))
BeforeDocRemoved(docID []byte, cb func(err error))
BeforeDocRolledBack(docID []byte, cb func(err error))
AfterDocRemovedPreRetry(docID []byte, cb func(err error))
AfterDocRemovedPostRetry(docID []byte, cb func(err error))
AfterGetComplete(docID []byte, cb func(err error))
AfterStagedReplaceComplete(docID []byte, cb func(err error))
AfterStagedRemoveComplete(docID []byte, cb func(err error))
AfterStagedInsertComplete(docID []byte, cb func(err error))
AfterRollbackReplaceOrRemove(docID []byte, cb func(err error))
AfterRollbackDeleteInserted(docID []byte, cb func(err error))
BeforeCheckATREntryForBlockingDoc(docID []byte, cb func(err error))
BeforeDocGet(docID []byte, cb func(err error))
BeforeGetDocInExistsDuringStagedInsert(docID []byte, cb func(err error))
BeforeRemoveStagedInsert(docID []byte, cb func(err error))
AfterRemoveStagedInsert(docID []byte, cb func(err error))
AfterDocsCommitted(func(err error))
AfterDocsRemoved(func(err error))
AfterATRPending(func(err error))
BeforeATRPending(func(err error))
BeforeATRComplete(func(err error))
BeforeATRRolledBack(func(err error))
AfterATRComplete(func(err error))
BeforeATRAborted(func(err error))
AfterATRAborted(func(err error))
AfterATRRolledBack(func(err error))
BeforeATRCommitAmbiguityResolution(func(err error))
RandomATRIDForVbucket(cb func(string, error))
HasExpiredClientSideHook(stage string, docID []byte, cb func(bool, error))
}
// TransactionCleanUpHooks provides a number of internal hooks used for testing.
// Internal: This should never be used and is not supported.
type TransactionCleanUpHooks interface {
BeforeATRGet(id []byte, cb func(error))
BeforeDocGet(id []byte, cb func(error))
BeforeRemoveLinks(id []byte, cb func(error))
BeforeCommitDoc(id []byte, cb func(error))
BeforeRemoveDocStagedForRemoval(id []byte, cb func(error))
BeforeRemoveDoc(id []byte, cb func(error))
BeforeATRRemove(id []byte, cb func(error))
}
// TransactionClientRecordHooks provides a number of internal hooks used for testing.
// Internal: This should never be used and is not supported.
type TransactionClientRecordHooks interface {
BeforeCreateRecord(cb func(error))
BeforeRemoveClient(cb func(error))
BeforeUpdateCAS(cb func(error))
BeforeGetRecord(cb func(error))
BeforeUpdateRecord(cb func(error))
}
// TransactionDefaultHooks is default set of noop hooks used within the library.
// Internal: This should never be used and is not supported.
type TransactionDefaultHooks struct {
}
// BeforeATRCommit occurs before an ATR is committed.
func (dh *TransactionDefaultHooks) BeforeATRCommit(cb func(err error)) {
cb(nil)
}
// AfterATRCommit occurs after an ATR is committed.
func (dh *TransactionDefaultHooks) AfterATRCommit(cb func(err error)) {
cb(nil)
}
// BeforeDocCommitted occurs before a document is committed.
func (dh *TransactionDefaultHooks) BeforeDocCommitted(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeRemovingDocDuringStagedInsert occurs before removing a document during staged insert.
func (dh *TransactionDefaultHooks) BeforeRemovingDocDuringStagedInsert(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeRollbackDeleteInserted occurs before rolling back a delete.
func (dh *TransactionDefaultHooks) BeforeRollbackDeleteInserted(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterDocCommittedBeforeSavingCAS occurs after committed a document before saving the CAS.
func (dh *TransactionDefaultHooks) AfterDocCommittedBeforeSavingCAS(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterDocCommitted occurs after a document is committed.
func (dh *TransactionDefaultHooks) AfterDocCommitted(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeStagedInsert occurs before staging an insert.
func (dh *TransactionDefaultHooks) BeforeStagedInsert(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeStagedRemove occurs before staging a remove.
func (dh *TransactionDefaultHooks) BeforeStagedRemove(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeStagedReplace occurs before staging a replace.
func (dh *TransactionDefaultHooks) BeforeStagedReplace(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeDocRemoved occurs before removing a document.
func (dh *TransactionDefaultHooks) BeforeDocRemoved(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeDocRolledBack occurs before a document is rolled back.
func (dh *TransactionDefaultHooks) BeforeDocRolledBack(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterDocRemovedPreRetry occurs after removing a document before retry.
func (dh *TransactionDefaultHooks) AfterDocRemovedPreRetry(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterDocRemovedPostRetry occurs after removing a document after retry.
func (dh *TransactionDefaultHooks) AfterDocRemovedPostRetry(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterGetComplete occurs after a get completes.
func (dh *TransactionDefaultHooks) AfterGetComplete(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterStagedReplaceComplete occurs after staging a replace is completed.
func (dh *TransactionDefaultHooks) AfterStagedReplaceComplete(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterStagedRemoveComplete occurs after staging a remove is completed.
func (dh *TransactionDefaultHooks) AfterStagedRemoveComplete(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterStagedInsertComplete occurs after staging an insert is completed.
func (dh *TransactionDefaultHooks) AfterStagedInsertComplete(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterRollbackReplaceOrRemove occurs after rolling back a replace or remove.
func (dh *TransactionDefaultHooks) AfterRollbackReplaceOrRemove(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterRollbackDeleteInserted occurs after rolling back a delete.
func (dh *TransactionDefaultHooks) AfterRollbackDeleteInserted(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeCheckATREntryForBlockingDoc occurs before checking the ATR of a blocking document.
func (dh *TransactionDefaultHooks) BeforeCheckATREntryForBlockingDoc(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeDocGet occurs before a document is fetched.
func (dh *TransactionDefaultHooks) BeforeDocGet(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeGetDocInExistsDuringStagedInsert occurs before getting a document for an insert.
func (dh *TransactionDefaultHooks) BeforeGetDocInExistsDuringStagedInsert(docID []byte, cb func(err error)) {
cb(nil)
}
// BeforeRemoveStagedInsert occurs before removing a staged insert.
func (dh *TransactionDefaultHooks) BeforeRemoveStagedInsert(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterRemoveStagedInsert occurs after removing a staged insert.
func (dh *TransactionDefaultHooks) AfterRemoveStagedInsert(docID []byte, cb func(err error)) {
cb(nil)
}
// AfterDocsCommitted occurs after all documents are committed.
func (dh *TransactionDefaultHooks) AfterDocsCommitted(cb func(err error)) {
cb(nil)
}
// AfterDocsRemoved occurs after all documents are removed.
func (dh *TransactionDefaultHooks) AfterDocsRemoved(cb func(err error)) {
cb(nil)
}
// AfterATRPending occurs after the ATR transitions to pending.
func (dh *TransactionDefaultHooks) AfterATRPending(cb func(err error)) {
cb(nil)
}
// BeforeATRPending occurs before the ATR transitions to pending.
func (dh *TransactionDefaultHooks) BeforeATRPending(cb func(err error)) {
cb(nil)
}
// BeforeATRComplete occurs before the ATR transitions to complete.
func (dh *TransactionDefaultHooks) BeforeATRComplete(cb func(err error)) {
cb(nil)
}
// BeforeATRRolledBack occurs before the ATR transitions to rolled back.
func (dh *TransactionDefaultHooks) BeforeATRRolledBack(cb func(err error)) {
cb(nil)
}
// AfterATRComplete occurs after the ATR transitions to complete.
func (dh *TransactionDefaultHooks) AfterATRComplete(cb func(err error)) {
cb(nil)
}
// BeforeATRAborted occurs before the ATR transitions to aborted.
func (dh *TransactionDefaultHooks) BeforeATRAborted(cb func(err error)) {
cb(nil)
}
// AfterATRAborted occurs after the ATR transitions to aborted.
func (dh *TransactionDefaultHooks) AfterATRAborted(cb func(err error)) {
cb(nil)
}
// AfterATRRolledBack occurs after the ATR transitions to rolled back.
func (dh *TransactionDefaultHooks) AfterATRRolledBack(cb func(err error)) {
cb(nil)
}
// BeforeATRCommitAmbiguityResolution occurs before ATR commit ambiguity resolution.
func (dh *TransactionDefaultHooks) BeforeATRCommitAmbiguityResolution(cb func(err error)) {
cb(nil)
}
// RandomATRIDForVbucket generates a random ATRID for a vbucket.
func (dh *TransactionDefaultHooks) RandomATRIDForVbucket(cb func(string, error)) {
cb("", nil)
}
// HasExpiredClientSideHook checks if a transaction has expired.
func (dh *TransactionDefaultHooks) HasExpiredClientSideHook(stage string, docID []byte, cb func(bool, error)) {
cb(false, nil)
}
// TransactionDefaultCleanupHooks is default set of noop hooks used within the library.
// Internal: This should never be used and is not supported.
type TransactionDefaultCleanupHooks struct {
}
// BeforeATRGet happens before an ATR get.
func (dh *TransactionDefaultCleanupHooks) BeforeATRGet(id []byte, cb func(error)) {
cb(nil)
}
// BeforeDocGet happens before an doc get.
func (dh *TransactionDefaultCleanupHooks) BeforeDocGet(id []byte, cb func(error)) {
cb(nil)
}
// BeforeRemoveLinks happens before we remove links.
func (dh *TransactionDefaultCleanupHooks) BeforeRemoveLinks(id []byte, cb func(error)) {
cb(nil)
}
// BeforeCommitDoc happens before we commit a document.
func (dh *TransactionDefaultCleanupHooks) BeforeCommitDoc(id []byte, cb func(error)) {
cb(nil)
}
// BeforeRemoveDocStagedForRemoval happens before we remove a staged document.
func (dh *TransactionDefaultCleanupHooks) BeforeRemoveDocStagedForRemoval(id []byte, cb func(error)) {
cb(nil)
}
// BeforeRemoveDoc happens before we remove a document.
func (dh *TransactionDefaultCleanupHooks) BeforeRemoveDoc(id []byte, cb func(error)) {
cb(nil)
}
// BeforeATRRemove happens before we remove an ATR.
func (dh *TransactionDefaultCleanupHooks) BeforeATRRemove(id []byte, cb func(error)) {
cb(nil)
}
// TransactionDefaultClientRecordHooks is default set of noop hooks used within the library.
// Internal: This should never be used and is not supported.
type TransactionDefaultClientRecordHooks struct {
}
// BeforeCreateRecord happens before we create a cleanup client record.
func (dh *TransactionDefaultClientRecordHooks) BeforeCreateRecord(cb func(error)) {
cb(nil)
}
// BeforeRemoveClient happens before we remove a cleanup client record.
func (dh *TransactionDefaultClientRecordHooks) BeforeRemoveClient(cb func(error)) {
cb(nil)
}
// BeforeUpdateCAS happens before we update a CAS.
func (dh *TransactionDefaultClientRecordHooks) BeforeUpdateCAS(cb func(error)) {
cb(nil)
}
// BeforeGetRecord happens before we get a cleanup client record.
func (dh *TransactionDefaultClientRecordHooks) BeforeGetRecord(cb func(error)) {
cb(nil)
}
// BeforeUpdateRecord happens before we update a cleanup client record.
func (dh *TransactionDefaultClientRecordHooks) BeforeUpdateRecord(cb func(error)) {
cb(nil)
}
// nolint: deadcode,varcheck
const (
hookRollback = "rollback"
hookGet = "get"
hookInsert = "insert"
hookReplace = "replace"
hookRemove = "remove"
hookCommit = "commit"
hookAbortGetATR = "abortGetAtr"
hookRollbackDoc = "rollbackDoc"
hookDeleteInserted = "deleteInserted"
hookCreateStagedInsert = "createdStagedInsert"
hookRemoveStagedInsert = "removeStagedInsert"
hookRemoveDoc = "removeDoc"
hookCommitDoc = "commitDoc"
hookWWC = "writeWriteConflict"
hookATRCommit = "atrCommit"
hookATRCommitAmbiguityResolution = "atrCommitAmbiguityResolution"
hookATRAbort = "atrAbort"
hookATRRollback = "atrRollbackComplete"
hookATRPending = "atrPending"
hookATRComplete = "atrComplete"
)
gocbcore-10.2.3/transactions_jsondata.go 0000664 0000000 0000000 00000007216 14417540156 0020317 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
)
type jsonAtrState string
const (
jsonAtrStateUnknown = jsonAtrState("")
jsonAtrStatePending = jsonAtrState("PENDING")
jsonAtrStateCommitted = jsonAtrState("COMMITTED")
jsonAtrStateCompleted = jsonAtrState("COMPLETED")
jsonAtrStateAborted = jsonAtrState("ABORTED")
jsonAtrStateRolledBack = jsonAtrState("ROLLED_BACK")
)
type jsonMutationType string
const (
jsonMutationInsert = jsonMutationType("insert")
jsonMutationReplace = jsonMutationType("replace")
jsonMutationRemove = jsonMutationType("remove")
)
type jsonAtrMutation struct {
BucketName string `json:"bkt,omitempty"`
ScopeName string `json:"scp,omitempty"`
CollectionName string `json:"col,omitempty"`
DocID string `json:"id,omitempty"`
}
type jsonAtrAttempt struct {
TransactionID string `json:"tid,omitempty"`
ExpiryTime uint `json:"exp,omitempty"`
State string `json:"st,omitempty"`
PendingCAS string `json:"tst,omitempty"`
CommitCAS string `json:"tsc,omitempty"`
CompletedCAS string `json:"tsco,omitempty"`
AbortCAS string `json:"tsrs,omitempty"`
RolledBackCAS string `json:"tsrc,omitempty"`
Inserts []jsonAtrMutation `json:"ins,omitempty"`
Replaces []jsonAtrMutation `json:"rep,omitempty"`
Removes []jsonAtrMutation `json:"rem,omitempty"`
DurabilityLevel string `json:"d,omitempty"`
ForwardCompat map[string][]jsonForwardCompatibilityEntry `json:"fc,omitempty"`
}
type jsonTxnXattrID struct {
Transaction string `json:"txn,omitempty"`
Attempt string `json:"atmpt,omitempty"`
}
type jsonTxnXattrATR struct {
DocID string `json:"id,omitempty"`
BucketName string `json:"bkt,omitempty"`
CollectionName string `json:"coll,omitempty"`
ScopeName string `json:"scp,omitempty"`
}
type jsonTxnXattrOp struct {
Type jsonMutationType `json:"type,omitempty"`
Staged json.RawMessage `json:"stgd,omitempty"`
CRC32 string `json:"crc32,omitempty"`
}
type jsonTxnXattrRestore struct {
OriginalCAS string `json:"CAS,omitempty"`
ExpiryTime uint `json:"exptime"`
RevID string `json:"revid,omitempty"`
}
type jsonTxnXattr struct {
ID jsonTxnXattrID `json:"id,omitempty"`
ATR jsonTxnXattrATR `json:"atr,omitempty"`
Operation jsonTxnXattrOp `json:"op,omitempty"`
Restore *jsonTxnXattrRestore `json:"restore,omitempty"`
ForwardCompat map[string][]jsonForwardCompatibilityEntry `json:"fc,omitempty"`
}
type transactionDocMeta struct {
Cas string `json:"CAS"`
RevID string `json:"revid"`
Expiration uint `json:"exptime"`
CRC32 string `json:"value_crc32c,omitempty"`
}
type transactionGetDoc struct {
Body []byte
TxnMeta *jsonTxnXattr
DocMeta *transactionDocMeta
Cas Cas
Deleted bool
}
type jsonForwardCompatibilityEntry struct {
ProtocolVersion string `json:"p,omitempty"`
ProtocolExtension string `json:"e,omitempty"`
Behaviour string `json:"b,omitempty"`
RetryInterval int `json:"ra,omitempty"`
}
gocbcore-10.2.3/transactions_lostcleanup.go 0000664 0000000 0000000 00000075345 14417540156 0021055 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"math"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/couchbase/gocbcore/v10/memd"
"github.com/google/uuid"
)
var clientRecordKey = []byte("_txn:client-record")
type jsonClientRecord struct {
HeartbeatMS string `json:"heartbeat_ms,omitempty"`
ExpiresMS int `json:"expires_ms,omitempty"`
NumATRs int `json:"num_atrs,omitempty"`
}
type jsonClientOverride struct {
Enabled bool `json:"enabled,omitempty"`
ExpiresNanos int64 `json:"expires,omitempty"`
}
type jsonClientRecords struct {
Clients map[string]jsonClientRecord `json:"clients"`
Override *jsonClientOverride `json:"override,omitempty"`
}
type jsonHLC struct {
NowSecs string `json:"now"`
}
// TransactionClientRecordDetails is the result of processing a client record.
// Internal: This should never be used and is not supported.
type TransactionClientRecordDetails struct {
NumActiveClients int
IndexOfThisClient int
ClientIsNew bool
ExpiredClientIDs []string
NumExistingClients int
NumExpiredClients int
OverrideEnabled bool
OverrideActive bool
OverrideExpiresCas int64
CasNowNanos int64
AtrsHandledByClient []string
CheckAtrEveryNMillis int
ClientUUID string
}
// TransactionProcessATRStats is the stats recorded when running a ProcessATR request.
// Internal: This should never be used and is not supported.
type TransactionProcessATRStats struct {
NumEntries int
NumEntriesExpired int
}
// LostTransactionCleaner is responsible for cleaning up lost transactions.
// Internal: This should never be used and is not supported.
type LostTransactionCleaner interface {
ProcessClient(agent *Agent, oboUser string, collection, scope, uuid string, cb func(*TransactionClientRecordDetails, error))
ProcessATR(agent *Agent, oboUser string, collection, scope, atrID string, cb func([]TransactionsCleanupAttempt, TransactionProcessATRStats, error))
RemoveClientFromAllLocations(uuid string) error
Close()
GetAndResetResourceUnits() *TransactionResourceUnitResult
}
type lostTransactionCleaner interface {
AddATRLocation(location TransactionLostATRLocation)
ATRLocations() []TransactionLostATRLocation
Close()
GetAndResetResourceUnits() *TransactionResourceUnitResult
}
type noopLostTransactionCleaner struct {
}
func (ltc *noopLostTransactionCleaner) AddATRLocation(location TransactionLostATRLocation) {
}
func (ltc *noopLostTransactionCleaner) ATRLocations() []TransactionLostATRLocation {
return nil
}
func (ltc *noopLostTransactionCleaner) Close() {
}
func (ltc *noopLostTransactionCleaner) GetAndResetResourceUnits() *TransactionResourceUnitResult {
return nil
}
type stdLostTransactionCleaner struct {
uuid string
cleanupHooks TransactionCleanUpHooks
clientRecordHooks TransactionClientRecordHooks
numAtrs int
cleanupWindow time.Duration
cleaner TransactionsCleaner
keyValueTimeout time.Duration
bucketAgentProvider TransactionsBucketAgentProviderFn
locations map[TransactionLostATRLocation]chan struct{}
locationsLock sync.Mutex
newLocationCh chan lostATRLocationWithShutdown
stop chan struct{}
atrLocationFinder TransactionsLostCleanupATRLocationProviderFn
numResourceUnitOps uint32
readUnits uint32
writeUnits uint32
}
type lostATRLocationWithShutdown struct {
location TransactionLostATRLocation
shutdown chan struct{}
}
// NewLostTransactionCleaner returns new lost transaction cleaner.
// Internal: This should never be used and is not supported.
func NewLostTransactionCleaner(config *TransactionsConfig) LostTransactionCleaner {
return newStdLostTransactionCleaner(config)
}
func newStdLostTransactionCleaner(config *TransactionsConfig) *stdLostTransactionCleaner {
return &stdLostTransactionCleaner{
uuid: uuid.New().String(),
numAtrs: config.Internal.NumATRs,
cleanupWindow: config.CleanupWindow,
cleanupHooks: config.Internal.CleanUpHooks,
clientRecordHooks: config.Internal.ClientRecordHooks,
cleaner: NewTransactionsCleaner(config),
keyValueTimeout: config.KeyValueTimeout,
bucketAgentProvider: config.BucketAgentProvider,
locations: make(map[TransactionLostATRLocation]chan struct{}),
newLocationCh: make(chan lostATRLocationWithShutdown, 20), // Buffer of 20 should be plenty
stop: make(chan struct{}),
atrLocationFinder: config.LostCleanupATRLocationProvider,
}
}
func startLostTransactionCleaner(config *TransactionsConfig) *stdLostTransactionCleaner {
t := newStdLostTransactionCleaner(config)
if config.BucketAgentProvider != nil {
go t.start()
}
return t
}
func (ltc *stdLostTransactionCleaner) start() {
logDebugf("Lost transactions %s starting", ltc.uuid)
ltc.fetchExtraCleanupLocations()
for {
select {
case <-ltc.stop:
return
case location := <-ltc.newLocationCh:
logDebugf("Starting new location handler for %s, location %s", ltc.uuid, location.location)
agent, oboUser, err := ltc.bucketAgentProvider(location.location.BucketName)
if err != nil {
logDebugf("Failed to fetch agent for %s, location: %s:, err: %v",
ltc.uuid, location.location, err)
// We should probably do something here...
continue
}
go ltc.perLocation(agent, oboUser, location)
}
}
}
func (ltc *stdLostTransactionCleaner) GetAndResetResourceUnits() *TransactionResourceUnitResult {
baseUnits := ltc.cleaner.GetAndResetResourceUnits()
numOps := atomic.SwapUint32(<c.numResourceUnitOps, 0)
if numOps == 0 && baseUnits == nil {
return nil
}
readUnits := atomic.SwapUint32(<c.readUnits, 0)
writeUnits := atomic.SwapUint32(<c.writeUnits, 0)
if baseUnits == nil {
return &TransactionResourceUnitResult{
NumOps: numOps,
ReadUnits: readUnits,
WriteUnits: writeUnits,
}
} else if numOps == 0 {
return baseUnits
}
return &TransactionResourceUnitResult{
NumOps: numOps + baseUnits.NumOps,
ReadUnits: readUnits + baseUnits.ReadUnits,
WriteUnits: writeUnits + baseUnits.WriteUnits,
}
}
func (ltc *stdLostTransactionCleaner) ATRLocations() []TransactionLostATRLocation {
ltc.locationsLock.Lock()
defer ltc.locationsLock.Unlock()
var locations []TransactionLostATRLocation
for location := range ltc.locations {
locations = append(locations, location)
}
return locations
}
func (ltc *stdLostTransactionCleaner) AddATRLocation(location TransactionLostATRLocation) {
ltc.locationsLock.Lock()
if _, ok := ltc.locations[location]; ok {
ltc.locationsLock.Unlock()
return
}
ch := make(chan struct{})
ltc.locations[location] = ch
ltc.locationsLock.Unlock()
logDebugf("Adding location %s to lost cleanup for %s", location, ltc.uuid)
ltc.newLocationCh <- lostATRLocationWithShutdown{
location: location,
shutdown: ch,
}
}
func (ltc *stdLostTransactionCleaner) Close() {
logDebugf("Lost transactions %s stopping", ltc.uuid)
close(ltc.stop)
err := ltc.RemoveClientFromAllLocations(ltc.uuid)
if err != nil {
logDebugf("Failed to remove client from all buckets: %v", err)
}
}
func (ltc *stdLostTransactionCleaner) RemoveClientFromAllLocations(uuid string) error {
logDebugf("Removing %s from all locations", ltc.uuid)
ltc.locationsLock.Lock()
locations := ltc.locations
ltc.locationsLock.Unlock()
if ltc.atrLocationFinder != nil {
bs, err := ltc.atrLocationFinder()
if err != nil {
logDebugf("Failed to get atr locations for %s: %v", ltc.uuid, err)
return err
}
for _, b := range bs {
if _, ok := locations[b]; !ok {
locations[b] = make(chan struct{})
}
}
}
return ltc.removeClient(uuid, locations)
}
func (ltc *stdLostTransactionCleaner) updateResourceUnits(units *ResourceUnitResult) {
if units == nil {
return
}
atomic.AddUint32(<c.numResourceUnitOps, 1)
atomic.AddUint32(<c.readUnits, uint32(units.ReadUnits))
atomic.AddUint32(<c.writeUnits, uint32(units.WriteUnits))
}
func (ltc *stdLostTransactionCleaner) updateResourceUnitsError(err error) {
if err == nil {
return
}
var kerr *KeyValueError
if errors.As(err, &kerr) {
ltc.updateResourceUnits(kerr.Internal.ResourceUnits)
}
}
func (ltc *stdLostTransactionCleaner) removeClient(uuid string, locations map[TransactionLostATRLocation]chan struct{}) error {
var err error
var wg sync.WaitGroup
for l := range locations {
wg.Add(1)
func(location TransactionLostATRLocation) {
// There's a possible race between here and the client record being updated/created.
// If that happens then it'll be expired and removed by another client anyway
deadline := time.Now().Add(500 * time.Millisecond)
ltc.unregisterClientRecord(location, uuid, deadline, func(unregErr error) {
if unregErr != nil {
logDebugf("Failed to unregister %s from cleanup record on from location %s: %v", uuid, location, unregErr)
err = unregErr
}
logInfof("Unregistered %s from cleanup record for location %v", uuid, location)
wg.Done()
})
}(l)
}
wg.Wait()
return err
}
func (ltc *stdLostTransactionCleaner) unregisterClientRecord(location TransactionLostATRLocation, uuid string, deadline time.Time, cb func(error)) {
logDebugf("Unregistering client %s for %s, location = %s", uuid, ltc.uuid, location)
agent, oboUser, err := ltc.bucketAgentProvider(location.BucketName)
if err != nil {
logDebugf("Failed to get agent for %s, location = %s, client = %s: %v", ltc.uuid, location, uuid, err)
select {
case <-time.After(time.Until(deadline)):
logDebugf("Timed out fetching agent for %s, location = %s, client = %s", ltc.uuid, location, uuid)
cb(ErrTimeout)
return
case <-time.After(10 * time.Millisecond):
}
ltc.unregisterClientRecord(location, uuid, deadline, cb)
return
}
ltc.clientRecordHooks.BeforeRemoveClient(func(err error) {
if err != nil {
if errors.Is(err, ErrDocumentNotFound) || errors.Is(err, ErrPathNotFound) {
cb(nil)
return
}
select {
case <-time.After(time.Until(deadline)):
cb(ErrTimeout)
return
case <-time.After(10 * time.Millisecond):
}
ltc.unregisterClientRecord(location, uuid, deadline, cb)
return
}
var opDeadline time.Time
if ltc.keyValueTimeout > 0 {
opDeadline = time.Now().Add(ltc.keyValueTimeout)
}
_, err = agent.MutateIn(MutateInOptions{
Key: clientRecordKey,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDelete,
Flags: memd.SubdocFlagXattrPath,
Path: "records.clients." + uuid,
},
},
Deadline: opDeadline,
CollectionName: location.CollectionName,
ScopeName: location.ScopeName,
User: oboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ltc.updateResourceUnitsError(err)
if errors.Is(err, ErrDocumentNotFound) || errors.Is(err, ErrPathNotFound) {
logDebugf("Client %s not found in client record for %s, location = %s: %v", uuid, ltc.uuid, location, err)
cb(nil)
return
}
logDebugf("Failed to remove client %s for %s, location = %s: %v", uuid, ltc.uuid, location, err)
go func() {
select {
case <-time.After(time.Until(deadline)):
logDebugf("Timed out removing client %s from client record for %s, location = %s", uuid, ltc.uuid, location)
cb(ErrTimeout)
return
case <-time.After(10 * time.Millisecond):
}
ltc.unregisterClientRecord(location, uuid, deadline, cb)
}()
return
}
ltc.updateResourceUnits(result.Internal.ResourceUnits)
cb(nil)
})
if err != nil {
logDebugf("Failed to schedule remove client %s for %s, location = %s: %v", uuid, ltc.uuid, location, err)
select {
case <-time.After(time.Until(deadline)):
logDebugf("Timed out scheduling client removal %s from client record for %s, location = %s", uuid,
ltc.uuid, location)
cb(ErrTimeout)
return
case <-time.After(10 * time.Millisecond):
}
ltc.unregisterClientRecord(location, uuid, deadline, cb)
}
})
}
func (ltc *stdLostTransactionCleaner) perLocation(agent *Agent, oboUser string, location lostATRLocationWithShutdown) {
logSchedf("Running cleanup %s on %s", ltc.uuid, location.location)
ltc.process(agent, oboUser, location.location.CollectionName, location.location.ScopeName, func(err error) {
if err != nil {
logDebugf("Cleanup failed for %s on %s", ltc.uuid, location.location)
// See comment in process for explanation of why we have a goroutine here.
go func() {
if errors.Is(err, ErrCollectionNotFound) {
logDebugf("Removing %s.%s.%s from lost cleanup %s due to collection no longer existing",
location.location.BucketName,
location.location.ScopeName,
location.location.CollectionName,
ltc.uuid,
)
close(location.shutdown) // This is unlikely to do anything as we're only listening here but best be safe.
ltc.locationsLock.Lock()
delete(ltc.locations, location.location)
ltc.locationsLock.Unlock()
return
}
select {
case <-ltc.stop:
return
case <-location.shutdown:
return
case <-time.After(1 * time.Second):
ltc.perLocation(agent, oboUser, location)
return
}
}()
return
}
select {
case <-ltc.stop:
return
case <-location.shutdown:
return
default:
}
ltc.perLocation(agent, oboUser, location)
})
}
func (ltc *stdLostTransactionCleaner) process(agent *Agent, oboUser string, collection, scope string, cb func(error)) {
ltc.ProcessClient(agent, oboUser, collection, scope, ltc.uuid, func(recordDetails *TransactionClientRecordDetails, err error) {
if err != nil {
logDebugf("Failed to process client %s on %s.%s.%s", ltc.uuid, agent.BucketName(), scope, collection)
var coreErr *TimeoutError
if errors.As(err, &coreErr) {
for _, reason := range coreErr.RetryReasons {
if reason == KVCollectionOutdatedRetryReason {
// We translate from outdated to not found here because at the point in time when we tried to
// use the collection it could not be found.
cb(ErrCollectionNotFound)
return
}
}
}
cb(err)
return
}
logDebugf("%s will check %d atrs, check every %d ms", ltc.uuid, len(recordDetails.AtrsHandledByClient),
recordDetails.CheckAtrEveryNMillis)
// We need this goroutine so we can release the scope of the callback. We're still in the callback from the
// LookupIn here so we're blocking the gocbcore read loop for the node, any further requests against that node
// will never complete and timeout.
go func() {
d := time.Duration(recordDetails.CheckAtrEveryNMillis) * time.Millisecond
for _, atr := range recordDetails.AtrsHandledByClient {
select {
case <-ltc.stop:
return
case <-time.After(d):
}
waitCh := make(chan error, 1)
ltc.ProcessATR(agent, oboUser, collection, scope, atr, func(attempts []TransactionsCleanupAttempt,
stats TransactionProcessATRStats, err error) {
waitCh <- err
})
err := <-waitCh
var coreErr *TimeoutError
if errors.As(err, &coreErr) {
for _, reason := range coreErr.RetryReasons {
if reason == KVCollectionOutdatedRetryReason {
cb(ErrCollectionNotFound)
return
}
}
}
}
cb(nil)
}()
})
}
// We pass uuid to this so that it's testable externally.
func (ltc *stdLostTransactionCleaner) ProcessClient(agent *Agent, oboUser string, collection, scope, uuid string, cb func(*TransactionClientRecordDetails, error)) {
logSchedf("Processing client %s for %s.%s.%s", uuid, agent.BucketName(), scope, collection)
ltc.clientRecordHooks.BeforeGetRecord(func(err error) {
if err != nil {
ec := classifyHookError(err)
switch ec.Class {
default:
cb(nil, err)
return
case TransactionErrorClassFailDocAlreadyExists:
case TransactionErrorClassFailCasMismatch:
}
}
var deadline time.Time
if ltc.keyValueTimeout > 0 {
deadline = time.Now().Add(ltc.keyValueTimeout)
}
_, err = agent.LookupIn(LookupInOptions{
Key: clientRecordKey,
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "records",
Flags: memd.SubdocFlagXattrPath,
},
{
Op: memd.SubDocOpGet,
Path: hlcMacro,
Flags: memd.SubdocFlagXattrPath,
},
},
Deadline: deadline,
CollectionName: collection,
ScopeName: scope,
User: oboUser,
}, func(result *LookupInResult, err error) {
if err != nil {
ltc.updateResourceUnitsError(err)
ec := classifyError(err)
switch ec.Class {
case TransactionErrorClassFailDocNotFound:
ltc.createClientRecord(agent, oboUser, collection, scope, func(err error) {
if err != nil {
logDebugf("%s failed to create client record: %v", ltc.uuid, err)
cb(nil, err)
return
}
ltc.ProcessClient(agent, oboUser, collection, scope, uuid, cb)
})
default:
cb(nil, err)
}
return
}
ltc.updateResourceUnits(result.Internal.ResourceUnits)
recordOp := result.Ops[0]
hlcOp := result.Ops[1]
if recordOp.Err != nil {
logDebugf("Failed to get records from client record for %s: %v", ltc.uuid, err)
cb(nil, recordOp.Err)
return
}
if hlcOp.Err != nil {
logDebugf("Failed to get hlc from client record for %s: %v", ltc.uuid, err)
cb(nil, hlcOp.Err)
return
}
var records jsonClientRecords
err = json.Unmarshal(recordOp.Value, &records)
if err != nil {
logDebugf("Failed to unmarshal records from client record for %s: %v", ltc.uuid, err)
cb(nil, err)
return
}
var hlc jsonHLC
err = json.Unmarshal(hlcOp.Value, &hlc)
if err != nil {
logDebugf("Failed to unmarshal hlc from client record for %s: %v", ltc.uuid, err)
cb(nil, err)
return
}
nowSecs, err := parseHLCToSeconds(hlc)
if err != nil {
logDebugf("Failed to parse hlc from client record for %s: %v", ltc.uuid, err)
cb(nil, err)
return
}
nowMS := nowSecs * 1000 // we need it in millis
recordDetails, err := ltc.parseClientRecords(records, uuid, nowMS)
if err != nil {
logDebugf("Failed to parse records from client record for %s: %v", ltc.uuid, err)
cb(nil, err)
return
}
if recordDetails.OverrideActive {
cb(&recordDetails, nil)
return
}
ltc.processClientRecord(agent, oboUser, collection, scope, uuid, recordDetails, func(err error) {
if err != nil {
logDebugf("%s failed to process client record %s: %v", ltc.uuid, uuid, err)
cb(nil, err)
return
}
cb(&recordDetails, nil)
})
})
if err != nil {
cb(nil, err)
return
}
})
}
func (ltc *stdLostTransactionCleaner) ProcessATR(agent *Agent, oboUser string, collection, scope, atrID string, cb func([]TransactionsCleanupAttempt, TransactionProcessATRStats, error)) {
ltc.getATR(agent, oboUser, collection, scope, atrID, func(attempts map[string]jsonAtrAttempt, hlc int64, err error) {
if err != nil {
// We want to be careful to not flood the logs with atr not found messages.
if !errors.Is(err, ErrDocumentNotFound) {
logSchedf("%s failed to get atr %s on %s.%s.%s", ltc.uuid, atrID, agent.BucketName(), scope, collection)
}
cb(nil, TransactionProcessATRStats{}, err)
return
}
if len(attempts) == 0 {
cb([]TransactionsCleanupAttempt{}, TransactionProcessATRStats{}, nil)
return
}
logSchedf("%s processing %d entries for atr %s on %s.%s.%s", ltc.uuid, len(attempts), atrID, agent.BucketName(), scope, collection)
stats := TransactionProcessATRStats{
NumEntries: len(attempts),
}
// See the explanation in process, same idea.
go func() {
var results []TransactionsCleanupAttempt
for key, attempt := range attempts {
select {
case <-ltc.stop:
return
default:
}
parsedCAS, err := parseCASToMilliseconds(attempt.PendingCAS)
if err != nil {
logDebugf("%s failed to parse CAS value %s for attempt %s on atr %s: %v", ltc.uuid,
attempt.PendingCAS, key, atrID, err)
cb(nil, TransactionProcessATRStats{}, err)
return
}
var inserts []TransactionsDocRecord
var replaces []TransactionsDocRecord
var removes []TransactionsDocRecord
for _, staged := range attempt.Inserts {
inserts = append(inserts, TransactionsDocRecord{
CollectionName: staged.CollectionName,
ScopeName: staged.ScopeName,
BucketName: staged.BucketName,
ID: []byte(staged.DocID),
})
}
for _, staged := range attempt.Replaces {
replaces = append(replaces, TransactionsDocRecord{
CollectionName: staged.CollectionName,
ScopeName: staged.ScopeName,
BucketName: staged.BucketName,
ID: []byte(staged.DocID),
})
}
for _, staged := range attempt.Removes {
removes = append(removes, TransactionsDocRecord{
CollectionName: staged.CollectionName,
ScopeName: staged.ScopeName,
BucketName: staged.BucketName,
ID: []byte(staged.DocID),
})
}
var st TransactionAttemptState
switch jsonAtrState(attempt.State) {
case jsonAtrStateCommitted:
st = TransactionAttemptStateCommitted
case jsonAtrStateCompleted:
st = TransactionAttemptStateCompleted
case jsonAtrStatePending:
st = TransactionAttemptStatePending
case jsonAtrStateAborted:
st = TransactionAttemptStateAborted
case jsonAtrStateRolledBack:
st = TransactionAttemptStateRolledBack
default:
continue
}
if int64(attempt.ExpiryTime)+parsedCAS < hlc {
logDebugf("%s detected expired attempt %s on atr %s", ltc.uuid, key, atrID)
req := &TransactionsCleanupRequest{
AttemptID: key,
AtrID: []byte(atrID),
AtrCollectionName: collection,
AtrScopeName: scope,
AtrBucketName: agent.BucketName(),
Inserts: inserts,
Replaces: replaces,
Removes: removes,
State: st,
ForwardCompat: jsonForwardCompatToForwardCompat(attempt.ForwardCompat),
DurabilityLevel: transactionsDurabilityLevelFromShorthand(attempt.DurabilityLevel),
Age: time.Duration(hlc - parsedCAS),
}
waitCh := make(chan TransactionsCleanupAttempt, 1)
ltc.cleaner.CleanupAttempt(agent, oboUser, req, false, func(attempt TransactionsCleanupAttempt) {
waitCh <- attempt
})
attempt := <-waitCh
results = append(results, attempt)
stats.NumEntriesExpired++
}
}
cb(results, stats, nil)
}()
})
}
func (ltc *stdLostTransactionCleaner) getATR(agent *Agent, oboUser string, collection, scope, atrID string,
cb func(map[string]jsonAtrAttempt, int64, error)) {
ltc.cleanupHooks.BeforeATRGet([]byte(atrID), func(err error) {
if err != nil {
cb(nil, 0, err)
return
}
var deadline time.Time
if ltc.keyValueTimeout > 0 {
deadline = time.Now().Add(ltc.keyValueTimeout)
}
_, err = agent.LookupIn(LookupInOptions{
Key: []byte(atrID),
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Path: "attempts",
Flags: memd.SubdocFlagXattrPath,
},
{
Op: memd.SubDocOpGet,
Path: hlcMacro,
Flags: memd.SubdocFlagXattrPath,
},
},
Deadline: deadline,
CollectionName: collection,
ScopeName: scope,
User: oboUser,
}, func(result *LookupInResult, err error) {
if err != nil {
ltc.updateResourceUnitsError(err)
cb(nil, 0, err)
return
}
ltc.updateResourceUnits(result.Internal.ResourceUnits)
if result.Ops[0].Err != nil {
cb(nil, 0, result.Ops[0].Err)
return
}
if result.Ops[1].Err != nil {
cb(nil, 0, result.Ops[1].Err)
return
}
var attempts map[string]jsonAtrAttempt
err = json.Unmarshal(result.Ops[0].Value, &attempts)
if err != nil {
cb(nil, 0, err)
return
}
var hlc jsonHLC
err = json.Unmarshal(result.Ops[1].Value, &hlc)
if err != nil {
cb(nil, 0, err)
return
}
nowSecs, err := parseHLCToSeconds(hlc)
if err != nil {
cb(nil, 0, err)
return
}
nowMS := nowSecs * 1000 // we need it in millis
cb(attempts, nowMS, err)
})
if err != nil {
cb(nil, 0, err)
return
}
})
}
func (ltc *stdLostTransactionCleaner) parseClientRecords(records jsonClientRecords, uuid string, hlc int64) (TransactionClientRecordDetails, error) {
var expiredIDs []string
var activeIDs []string
var clientAlreadyExists bool
for u, client := range records.Clients {
if u == uuid {
activeIDs = append(activeIDs, u)
clientAlreadyExists = true
continue
}
heartbeatMS, err := parseCASToMilliseconds(client.HeartbeatMS)
if err != nil {
return TransactionClientRecordDetails{}, err
}
expiredPeriod := hlc - heartbeatMS
if expiredPeriod >= int64(client.ExpiresMS) {
expiredIDs = append(expiredIDs, u)
} else {
activeIDs = append(activeIDs, u)
}
}
if !clientAlreadyExists {
activeIDs = append(activeIDs, uuid)
}
sort.Strings(activeIDs)
clientIndex := 0
for i, u := range activeIDs {
if u == uuid {
clientIndex = i
break
}
}
var overrideEnabled bool
var overrideActive bool
var overrideExpiresCas int64
if records.Override != nil {
overrideEnabled = records.Override.Enabled
overrideExpiresCas = records.Override.ExpiresNanos
hlcNanos := hlc * 1000000
if overrideEnabled && overrideExpiresCas > hlcNanos {
overrideActive = true
}
}
numActive := len(activeIDs)
numExpired := len(expiredIDs)
atrsHandled := atrsToHandle(clientIndex, numActive, ltc.numAtrs)
checkAtrEveryNS := ltc.cleanupWindow.Milliseconds() / int64(len(atrsHandled))
checkAtrEveryNMS := int(math.Max(1, float64(checkAtrEveryNS)))
return TransactionClientRecordDetails{
NumActiveClients: numActive,
IndexOfThisClient: clientIndex,
ClientIsNew: clientAlreadyExists,
ExpiredClientIDs: expiredIDs,
NumExistingClients: numActive + numExpired,
NumExpiredClients: numExpired,
OverrideEnabled: overrideEnabled,
OverrideActive: overrideActive,
OverrideExpiresCas: overrideExpiresCas,
CasNowNanos: hlc,
AtrsHandledByClient: atrsHandled,
CheckAtrEveryNMillis: checkAtrEveryNMS,
ClientUUID: uuid,
}, nil
}
func (ltc *stdLostTransactionCleaner) processClientRecord(agent *Agent, oboUser string, collection, scope, uuid string,
recordDetails TransactionClientRecordDetails, cb func(error)) {
logSchedf("%s processing client record %s for %s.%s.%s", ltc.uuid, uuid, agent.BucketName(), scope, collection)
ltc.clientRecordHooks.BeforeUpdateRecord(func(err error) {
if err != nil {
cb(err)
return
}
prefix := "records.clients." + uuid + "."
var marshalErr error
fieldOp := func(fieldName string, data interface{}, op memd.SubDocOpType, flags memd.SubdocFlag) SubDocOp {
b, err := json.Marshal(data)
if err != nil {
marshalErr = err
return SubDocOp{}
}
return SubDocOp{
Op: op,
Flags: flags,
Path: prefix + fieldName,
Value: b,
}
}
if marshalErr != nil {
cb(err)
return
}
ops := []SubDocOp{
fieldOp("heartbeat_ms", "${Mutation.CAS}", memd.SubDocOpDictSet,
memd.SubdocFlagXattrPath|memd.SubdocFlagExpandMacros|memd.SubdocFlagMkDirP),
fieldOp("expires_ms", (ltc.cleanupWindow + 20000*time.Millisecond).Milliseconds(),
memd.SubDocOpDictSet, memd.SubdocFlagXattrPath),
fieldOp("num_atrs", ltc.numAtrs, memd.SubDocOpDictSet, memd.SubdocFlagXattrPath),
{
Op: memd.SubDocOpSetDoc,
Flags: memd.SubdocFlagNone,
Value: []byte{0},
},
}
numOps := 12
if len(recordDetails.ExpiredClientIDs) < 12 {
numOps = len(recordDetails.ExpiredClientIDs)
}
for i := 0; i < numOps; i++ {
ops = append(ops, SubDocOp{
Op: memd.SubDocOpDelete,
Flags: memd.SubdocFlagXattrPath,
Path: "records.clients." + recordDetails.ExpiredClientIDs[i],
})
}
deadline := time.Time{}
if ltc.keyValueTimeout > 0 {
deadline = time.Now().Add(ltc.keyValueTimeout)
}
_, err = agent.MutateIn(MutateInOptions{
Key: clientRecordKey,
Ops: ops,
CollectionName: collection,
ScopeName: scope,
Deadline: deadline,
User: oboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ltc.updateResourceUnitsError(err)
cb(err)
return
}
ltc.updateResourceUnits(result.Internal.ResourceUnits)
cb(nil)
})
if err != nil {
cb(err)
return
}
})
}
func (ltc *stdLostTransactionCleaner) createClientRecord(agent *Agent, oboUser string, collection, scope string, cb func(error)) {
logDebugf("%s creating client record in %s.%s.%s", ltc.uuid, agent.BucketName(), scope, collection)
ltc.clientRecordHooks.BeforeCreateRecord(func(err error) {
if err != nil {
ec := classifyHookError(err)
switch ec.Class {
default:
cb(err)
return
case TransactionErrorClassFailDocNotFound:
}
}
var deadline time.Time
if ltc.keyValueTimeout > 0 {
deadline = time.Now().Add(ltc.keyValueTimeout)
}
_, err = agent.MutateIn(MutateInOptions{
Key: clientRecordKey,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictAdd,
Flags: memd.SubdocFlagXattrPath,
Path: "records.clients",
Value: []byte{123, 125}, // {}
},
{
Op: memd.SubDocOpSetDoc,
Flags: memd.SubdocFlagNone,
Path: "",
Value: []byte{0},
},
},
Flags: memd.SubdocDocFlagAddDoc,
Deadline: deadline,
CollectionName: collection,
ScopeName: scope,
User: oboUser,
}, func(result *MutateInResult, err error) {
if err != nil {
ltc.updateResourceUnitsError(err)
ec := classifyError(err)
switch ec.Class {
default:
cb(err)
return
case TransactionErrorClassFailDocAlreadyExists:
case TransactionErrorClassFailCasMismatch:
}
cb(nil)
return
}
ltc.updateResourceUnits(result.Internal.ResourceUnits)
cb(nil)
})
if err != nil {
cb(err)
return
}
})
}
func (ltc *stdLostTransactionCleaner) fetchExtraCleanupLocations() {
if ltc.atrLocationFinder != nil {
locations, err := ltc.atrLocationFinder()
if err != nil {
logDebugf("%s failed to fetch extra cleanup locations: %v", ltc.uuid, err)
return
}
locationMap := make(map[TransactionLostATRLocation]struct{})
for _, location := range locations {
ltc.AddATRLocation(location)
locationMap[location] = struct{}{}
}
}
}
func atrsToHandle(index int, numActive int, numAtrs int) []string {
allAtrs := transactionAtrIDList[:numAtrs]
var selectedAtrs []string
for i := index; i < len(allAtrs); i += numActive {
selectedAtrs = append(selectedAtrs, allAtrs[i])
}
return selectedAtrs
}
gocbcore-10.2.3/transactions_lostcleanup_test.go 0000664 0000000 0000000 00000027565 14417540156 0022115 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
"log"
"time"
"github.com/google/uuid"
"github.com/couchbase/gocbcore/v10/memd"
)
func (suite *StandardTestSuite) buildCleaner(agent *Agent, numATRs int,
locations map[TransactionLostATRLocation]chan struct{}) *stdLostTransactionCleaner {
clientUUID := uuid.New().String()
config := &TransactionsConfig{}
config.DurabilityLevel = TransactionDurabilityLevelNone
config.BucketAgentProvider = func(bucketName string) (*Agent, string, error) {
// We can always return just this one agent as we only actually
// use a single bucket for this entire test.
return agent, "", nil
}
config.CleanupWindow = 1 * time.Second
config.ExpirationTime = 500 * time.Millisecond
config.KeyValueTimeout = 2500 * time.Millisecond
config.Internal.Hooks = nil
config.Internal.CleanUpHooks = &TransactionDefaultCleanupHooks{}
config.Internal.ClientRecordHooks = &TransactionDefaultClientRecordHooks{}
config.Internal.NumATRs = numATRs
cleaner := newStdLostTransactionCleaner(config)
cleaner.locations = locations
cleaner.uuid = clientUUID
return cleaner
}
func (suite *UnitTestSuite) TestParseCas() {
// assertEquals(1539336197457L, ActiveTransactionRecord.parseMutationCAS("0x000058a71dd25c15"));
cas, err := parseCASToMilliseconds("0x000058a71dd25c15")
suite.Require().Nil(err)
suite.Require().Equal(int64(1539336197457), cas)
}
func (suite *StandardTestSuite) TestLostCleanupProcessClientSuccessfulTxn() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, s := suite.GetAgentAndTxnHarness()
h := suite.GetHarness()
h.PushOp(agent.Delete(DeleteOptions{
Key: clientRecordKey,
}, func(result *DeleteResult, err error) {
h.Wrap(func() {
if err != nil && !errors.Is(err, ErrDocumentNotFound) {
s.Fatalf("Remove operation failed: %v", err)
}
})
}))
h.Wait(0)
transactions, err := InitTransactions(&TransactionsConfig{
DurabilityLevel: TransactionDurabilityLevelNone,
BucketAgentProvider: func(bucketName string) (*Agent, string, error) {
// We can always return just this one agent as we only actually
// use a single bucket for this entire test.
return agent, "", nil
},
ExpirationTime: 500 * time.Millisecond,
KeyValueTimeout: 2500 * time.Millisecond,
CleanupLostAttempts: false,
})
if err != nil {
log.Printf("InitTransactions failed: %+v", err)
panic(err)
}
cleaner := suite.buildCleaner(agent, 1, map[TransactionLostATRLocation]chan struct{}{
{
BucketName: agent.BucketName(),
}: make(chan struct{}),
})
cleaner.process(agent, "", "", "", func(err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("process operation failed: %v", err)
}
})
})
s.Wait(0)
if suite.SupportsFeature(TestFeatureResourceUnits) {
units := cleaner.GetAndResetResourceUnits()
if suite.Assert().NotNil(units) {
suite.Assert().Greater(units.ReadUnits, uint16(0))
suite.Assert().Greater(units.WriteUnits, uint16(0))
suite.Assert().Greater(units.NumOps, uint32(0))
}
}
// Ensure that this cleaner has added itself to the client record
ops := []SubDocOp{
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagXattrPath,
Path: "records.clients." + cleaner.uuid,
},
}
h.PushOp(agent.LookupIn(LookupInOptions{
Key: clientRecordKey,
Ops: ops,
}, func(res *LookupInResult, err error) {
h.Wrap(func() {
if err != nil {
h.Fatalf("Lookup operation failed: %v", err)
}
if res.Ops[0].Err != nil {
h.Fatalf("Lookup operation 0 should not have failed, was: %v", res.Ops[0].Err)
}
})
}))
h.Wait(0)
suite.Require().Nil(transactions.Close())
cleaner.Close()
// Ensure that the cleaner has removed itself from the client record.
ops = []SubDocOp{
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagXattrPath,
Path: "records",
},
}
h.PushOp(agent.LookupIn(LookupInOptions{
Key: clientRecordKey,
Ops: ops,
}, func(res *LookupInResult, err error) {
h.Wrap(func() {
if err != nil {
h.Fatalf("Lookup operation failed: %v", err)
}
if res.Ops[0].Err != nil {
h.Fatalf("Lookup operation 0 failed: %v", res.Ops[0].Err)
}
var resultingClients jsonClientRecords
if err := json.Unmarshal(res.Ops[0].Value, &resultingClients); err != nil {
h.Fatalf("Unmarshal operation failed: %v", err)
}
if len(resultingClients.Clients) != 0 {
h.Fatalf("Client records should have been empty: %v", resultingClients)
}
})
}))
h.Wait(0)
}
func (suite *StandardTestSuite) TestLostCleanupCleansUpExpiredClients() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, s := suite.GetAgentAndTxnHarness()
h := suite.GetHarness()
// Create an expired client record that will be cleaned up
expiredClientID := uuid.New().String()
newClient := jsonClientRecords{
Clients: map[string]jsonClientRecord{
expiredClientID: {
HeartbeatMS: "0x000056a9039a4416",
ExpiresMS: 1,
},
},
}
b, err := json.Marshal(newClient)
suite.Require().Nil(err)
h.PushOp(agent.MutateIn(MutateInOptions{
Key: clientRecordKey,
Ops: []SubDocOp{
{
Op: memd.SubDocOpDictSet,
Flags: memd.SubdocFlagXattrPath | memd.SubdocFlagMkDirP,
Path: "records",
Value: b,
},
},
Flags: memd.SubdocDocFlagMkDoc,
}, func(result *MutateInResult, err error) {
h.Wrap(func() {
if err != nil {
s.Fatalf("MutateIn operation failed: %v", err)
}
})
}))
h.Wait(0)
cleaner := suite.buildCleaner(agent, 1024, nil)
cleaner.ProcessClient(agent, "", "", "", cleaner.uuid, func(details *TransactionClientRecordDetails, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("ProcessClient operation failed: %v", err)
}
})
})
s.Wait(0)
ops := []SubDocOp{
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagXattrPath,
Path: "records",
},
}
h.PushOp(agent.LookupIn(LookupInOptions{
Key: clientRecordKey,
Ops: ops,
}, func(res *LookupInResult, err error) {
h.Wrap(func() {
if err != nil {
h.Fatalf("Lookup operation failed: %v", err)
}
if res.Ops[0].Err != nil {
h.Fatalf("Lookup operation 0 failed: %v", err)
}
var resultingClients jsonClientRecords
if err := json.Unmarshal(res.Ops[0].Value, &resultingClients); err != nil {
h.Fatalf("Unmarshal failed: %v", err)
}
if len(resultingClients.Clients) != 1 {
h.Fatalf("Expected client records to have 1 client: %v", resultingClients)
}
if _, ok := resultingClients.Clients[expiredClientID]; ok {
h.Fatalf("Expected client records not contain old client id %s: %v", expiredClientID, resultingClients)
}
})
}))
h.Wait(0)
}
type abortATRHooks struct {
*TransactionDefaultHooks
}
func (h *abortATRHooks) BeforeATRAborted(cb func(err error)) {
cb(errors.New("some error"))
}
func (suite *StandardTestSuite) TestLostCleanupProcessRollback() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, s := suite.GetAgentAndTxnHarness()
snap, err := agent.ConfigSnapshot()
suite.Require().Nil(err, err)
numSrvrs, err := snap.NumServers()
suite.Require().Nil(err, err)
if numSrvrs == 1 {
suite.T().Skip("Skipping test due to only 1 server, durability used by cleanup not possible")
}
h := suite.GetHarness()
h.PushOp(agent.Delete(DeleteOptions{
Key: clientRecordKey,
}, func(result *DeleteResult, err error) {
h.Wrap(func() {
if err != nil && !errors.Is(err, ErrDocumentNotFound) {
s.Fatalf("Remove operation failed: %v", err)
}
})
}))
h.Wait(0)
cfg := &TransactionsConfig{
DurabilityLevel: TransactionDurabilityLevelNone,
BucketAgentProvider: func(bucketName string) (*Agent, string, error) {
// We can always return just this one agent as we only actually
// use a single bucket for this entire test.
return agent, "", nil
},
ExpirationTime: 500 * time.Millisecond,
KeyValueTimeout: 2500 * time.Millisecond,
CleanupLostAttempts: false,
}
cfg.Internal.Hooks = &abortATRHooks{}
transactions, err := InitTransactions(cfg)
if err != nil {
log.Printf("InitTransactions failed: %+v", err)
panic(err)
}
txn, err := transactions.BeginTransaction(nil)
suite.Require().Nil(err, err)
val1 := []byte(`{"name":"mike"}`)
key := uuid.NewString()
// Start the attempt
err = txn.NewAttempt()
suite.Require().Nil(err, err)
_, err = testBlkInsert(txn, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(key),
Value: val1,
})
suite.Require().Nil(err, "insert failed")
err = testBlkRollback(txn)
suite.Require().NotNil(err)
cleaner := suite.buildCleaner(agent, 1, map[TransactionLostATRLocation]chan struct{}{
{
BucketName: agent.BucketName(),
}: make(chan struct{}),
})
cleaner.ProcessClient(agent, "", suite.CollectionName, suite.ScopeName, cleaner.uuid, func(result *TransactionClientRecordDetails, err error) {
s.Wrap(func() {
if err != nil {
s.Fatalf("process client operation failed: %v", err)
}
})
})
s.Wait(0)
success := suite.tryUntil(time.Now().Add(2*time.Second), 250*time.Millisecond, func() bool {
wait := make(chan struct {
err error
attempts []TransactionsCleanupAttempt
stats TransactionProcessATRStats
}, 1)
cleaner.ProcessATR(agent, "", txn.Attempt().AtrCollectionName, txn.Attempt().AtrScopeName, string(txn.Attempt().AtrID),
func(attempts []TransactionsCleanupAttempt, stats TransactionProcessATRStats, err error) {
wait <- struct {
err error
attempts []TransactionsCleanupAttempt
stats TransactionProcessATRStats
}{err: err, attempts: attempts, stats: stats}
})
res := <-wait
if len(res.attempts) == 0 {
return false
}
if !res.attempts[0].Success {
return false
}
if res.stats.NumEntriesExpired == 0 {
return false
}
return true
})
suite.Require().True(success, "ProcessATR did not succeed in time")
if suite.SupportsFeature(TestFeatureResourceUnits) {
units := cleaner.GetAndResetResourceUnits()
if suite.Assert().NotNil(units) {
suite.Assert().Greater(units.ReadUnits, uint16(0))
suite.Assert().Greater(units.WriteUnits, uint16(0))
suite.Assert().Greater(units.NumOps, uint32(0))
}
}
suite.Require().Nil(transactions.Close())
cleaner.Close()
}
func (suite *StandardTestSuite) TestCustomATRLocationAutomaticallyAddedToCleanup() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, _ := suite.GetAgentAndTxnHarness()
loc := TransactionATRLocation{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}
cfg := &TransactionsConfig{
DurabilityLevel: TransactionDurabilityLevelNone,
BucketAgentProvider: func(bucketName string) (*Agent, string, error) {
// We can always return just this one agent as we only actually
// use a single bucket for this entire test.
return agent, "", nil
},
ExpirationTime: 500 * time.Millisecond,
KeyValueTimeout: 2500 * time.Millisecond,
CleanupLostAttempts: true,
CustomATRLocation: loc,
}
transactions, err := InitTransactions(cfg)
if err != nil {
log.Printf("InitTransactions failed: %+v", err)
panic(err)
}
defer transactions.Close()
suite.Require().Eventually(func() bool {
locs := transactions.Internal().CleanupLocations()
if len(locs) == 0 {
return false
}
cLoc := locs[0]
return cLoc.BucketName == loc.Agent.BucketName() && cLoc.ScopeName == loc.ScopeName &&
cLoc.CollectionName == loc.CollectionName
}, 5*time.Second, 100*time.Millisecond)
}
gocbcore-10.2.3/transactions_serializedcontext.go 0000664 0000000 0000000 00000002643 14417540156 0022253 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
type jsonSerializedMutation struct {
Bucket string `json:"bkt"`
Scope string `json:"scp"`
Collection string `json:"coll"`
ID string `json:"id"`
Cas string `json:"cas"`
Type string `json:"type"`
}
type jsonSerializedAttempt struct {
ID struct {
Transaction string `json:"txn"`
Attempt string `json:"atmpt"`
} `json:"id"`
ATR struct {
Bucket string `json:"bkt"`
Scope string `json:"scp"`
Collection string `json:"coll"`
ID string `json:"id"`
} `json:"atr"`
Config struct {
KeyValueTimeoutMs int `json:"kvTimeoutMs"`
DurabilityLevel string `json:"durabilityLevel"`
NumAtrs int `json:"numAtrs"`
} `json:"config"`
State struct {
TimeLeftMs int `json:"timeLeftMs"`
} `json:"state"`
Mutations []jsonSerializedMutation `json:"mutations"`
}
gocbcore-10.2.3/transactions_stagedmutation.go 0000664 0000000 0000000 00000005354 14417540156 0021545 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"encoding/json"
"errors"
)
// TransactionStagedMutationType represents the type of a mutation performed in a transaction.
type TransactionStagedMutationType int
const (
// TransactionStagedMutationUnknown indicates an error has occured.
TransactionStagedMutationUnknown = TransactionStagedMutationType(0)
// TransactionStagedMutationInsert indicates the staged mutation was an insert operation.
TransactionStagedMutationInsert = TransactionStagedMutationType(1)
// TransactionStagedMutationReplace indicates the staged mutation was an replace operation.
TransactionStagedMutationReplace = TransactionStagedMutationType(2)
// TransactionStagedMutationRemove indicates the staged mutation was an remove operation.
TransactionStagedMutationRemove = TransactionStagedMutationType(3)
)
func transactionStagedMutationTypeToString(mtype TransactionStagedMutationType) string {
switch mtype {
case TransactionStagedMutationInsert:
return "INSERT"
case TransactionStagedMutationReplace:
return "REPLACE"
case TransactionStagedMutationRemove:
return "REMOVE"
}
return ""
}
func transactionStagedMutationTypeFromString(mtype string) (TransactionStagedMutationType, error) {
switch mtype {
case "INSERT":
return TransactionStagedMutationInsert, nil
case "REPLACE":
return TransactionStagedMutationReplace, nil
case "REMOVE":
return TransactionStagedMutationRemove, nil
}
return TransactionStagedMutationUnknown, errors.New("invalid mutation type string")
}
// TransactionStagedMutation wraps all of the information about a mutation which has been staged
// as part of the transaction and which should later be unstaged when the transaction
// has been committed.
type TransactionStagedMutation struct {
OpType TransactionStagedMutationType
BucketName string
ScopeName string
CollectionName string
Key []byte
Cas Cas
Staged json.RawMessage
}
type transactionStagedMutation struct {
OpType TransactionStagedMutationType
Agent *Agent
OboUser string
ScopeName string
CollectionName string
Key []byte
Cas Cas
Staged json.RawMessage
}
gocbcore-10.2.3/transactions_test.go 0000664 0000000 0000000 00000106611 14417540156 0017472 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"errors"
"fmt"
"github.com/google/uuid"
"log"
"strings"
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func (suite *StandardTestSuite) fetchStagedOpData(key string, agent *Agent, opHarness *TestSubHarness) (jsonMutationType, []byte, bool) {
var res *LookupInResult
var err error
opHarness.PushOp(agent.LookupIn(LookupInOptions{
Key: []byte(key),
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagXattrPath,
Path: "txn.op.type",
},
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagXattrPath,
Path: "txn.op.stgd",
},
},
Flags: memd.SubdocDocFlagAccessDeleted,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *LookupInResult, opErr error) {
opHarness.Wrap(func() {
err = opErr
res = result
})
}))
opHarness.Wait(0)
if err != nil {
return "", nil, false
}
var opType string
err = json.Unmarshal(res.Ops[0].Value, &opType)
if err != nil {
return "", nil, false
}
stgdData := res.Ops[1].Value
var exists bool
opHarness.PushOp(agent.GetMeta(GetMetaOptions{
Key: []byte(key),
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *GetMetaResult, err error) {
opHarness.Wrap(func() {
if err != nil {
exists = false
return
}
exists = result.Deleted == 0
})
}))
opHarness.Wait(0)
return jsonMutationType(opType), stgdData, exists
}
func (suite *StandardTestSuite) assertStagedDoc(key string, expOpType jsonMutationType, expStgdData []byte,
expTombstone bool, agent *Agent, opHarness *TestSubHarness) {
stgdOpType, stgdData, docExists := suite.fetchStagedOpData(key, agent, opHarness)
suite.Assert().Equal(
expOpType,
stgdOpType,
fmt.Sprintf("%s had an incorrect op type", key))
suite.Assert().Equal(
expStgdData,
stgdData,
fmt.Sprintf("%s had an incorrect staged data", key))
suite.Assert().Equal(
expTombstone,
!docExists,
fmt.Sprintf("%s document state did not match", key))
}
func (suite *StandardTestSuite) assertDocNotStaged(key string, agent *Agent, opHarness *TestSubHarness) {
var res *LookupInResult
var err error
opHarness.PushOp(agent.LookupIn(LookupInOptions{
Key: []byte(key),
Ops: []SubDocOp{
{
Op: memd.SubDocOpGet,
Flags: memd.SubdocFlagXattrPath,
Path: "txn",
},
},
Flags: memd.SubdocDocFlagAccessDeleted,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *LookupInResult, opErr error) {
opHarness.Wrap(func() {
err = opErr
res = result
})
}))
opHarness.Wait(0)
suite.Require().Nil(err, err)
suite.Require().True(errors.Is(res.Ops[0].Err, ErrPathNotFound))
}
func (suite *StandardTestSuite) initTransactionAndAttempt(agent *Agent) (*TransactionsManager, *Transaction) {
txns, err := InitTransactions(&TransactionsConfig{
DurabilityLevel: TransactionDurabilityLevelNone,
BucketAgentProvider: func(bucketName string) (*Agent, string, error) {
// We can always return just this one agent as we only actually
// use a single bucket for this entire test.
return agent, "", nil
},
ExpirationTime: 60 * time.Second,
})
suite.Require().Nil(err, err)
txn, err := txns.BeginTransaction(nil)
suite.Require().Nil(err, err)
// Start the attempt
err = txn.NewAttempt()
suite.Require().Nil(err, err)
return txns, txn
}
func (suite *StandardTestSuite) TestTransactionsInsertTxn1GetTxn2() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy2 := []byte(`{"name":"mike"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
_, err := testBlkInsert(txn, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`insertDoc`),
Value: testDummy2,
})
suite.Require().Nil(err, "insert of insertDoc failed")
txn = suite.serializeUnserializeTxn(txns, txn)
txn2, err := txns.BeginTransaction(nil)
suite.Require().Nil(err, "txn2 begin failed")
err = txn2.NewAttempt()
suite.Require().Nil(err, "txn2 attempt start failed")
_, err = testBlkGet(txn2, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`insertDoc`),
})
suite.Require().True(errors.Is(err, ErrDocumentNotFound), "insertDoc get from T2 should have failed")
suite.assertStagedDoc("insertDoc", jsonMutationInsert, testDummy2, true, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("insertDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsReplaceTxn1GetTxn2() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy1 := []byte(`{"name":"joel"}`)
testDummy2 := []byte(`{"name":"mike"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
opHarness.PushOp(agent.Set(SetOptions{
Key: []byte("replaceDoc"),
Value: testDummy1,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *StoreResult, err error) {
opHarness.Wrap(func() {
if err != nil {
opHarness.Fatalf("Set command failed: %v", err)
}
})
}))
opHarness.Wait(0)
replaceGetRes, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`replaceDoc`),
})
suite.Require().Nil(err, "replaceDoc get failed")
_, err = testBlkReplace(txn, TransactionReplaceOptions{
Document: replaceGetRes,
Value: testDummy2,
})
suite.Require().Nil(err, "replaceDoc replace failed")
txn = suite.serializeUnserializeTxn(txns, txn)
txn2, err := txns.BeginTransaction(nil)
suite.Require().Nil(err, "txn2 begin failed")
err = txn2.NewAttempt()
suite.Require().Nil(err, "txn2 attempt start failed")
getOfReplace, err := testBlkGet(txn2, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`replaceDoc`),
})
suite.Require().Nil(err, "replaceDoc get from T2 should have succeeded")
suite.Assert().Equal(testDummy1, getOfReplace.Value, "replaceDoc get from T2 should have right data")
suite.assertStagedDoc("replaceDoc", jsonMutationReplace, testDummy2, false, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("replaceDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsRemoveTxn1GetTxn2() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy1 := []byte(`{"name":"joel"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
opHarness.PushOp(agent.Set(SetOptions{
Key: []byte("removeDoc"),
Value: testDummy1,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *StoreResult, err error) {
opHarness.Wrap(func() {
if err != nil {
opHarness.Fatalf("Set command failed: %v", err)
}
})
}))
opHarness.Wait(0)
removeGetRes, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`removeDoc`),
})
suite.Require().Nil(err, "removeDoc get failed")
log.Printf("removeDoc get result: %+v", removeGetRes)
removeRes, err := testBlkRemove(txn, TransactionRemoveOptions{
Document: removeGetRes,
})
suite.Require().Nil(err, "removeRes remove failed")
log.Printf("removeRes remove result: %+v", removeRes)
txn = suite.serializeUnserializeTxn(txns, txn)
txn2, err := txns.BeginTransaction(nil)
suite.Require().Nil(err, "txn2 begin failed")
err = txn2.NewAttempt()
suite.Require().Nil(err, "txn2 attempt start failed")
getOfRemove, err := testBlkGet(txn2, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`removeDoc`),
})
suite.Require().Nil(err, "removeDoc get from T2 should have succeeded")
suite.Assert().Equal(testDummy1, getOfRemove.Value, "removeDoc get from T2 should have right data")
suite.assertStagedDoc("removeDoc", jsonMutationRemove, []byte{}, false, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("removeDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsReplaceTxn1InsertTxn2() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy1 := []byte(`{"name":"joel"}`)
testDummy2 := []byte(`{"name":"mike"}`)
testDummy3 := []byte(`{"name":"frank"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
opHarness.PushOp(agent.Set(SetOptions{
Key: []byte("replaceToInsertDoc"),
Value: testDummy1,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *StoreResult, err error) {
opHarness.Wrap(func() {
if err != nil {
opHarness.Fatalf("Set command failed: %v", err)
}
})
}))
opHarness.Wait(0)
replaceToInsertGetRes, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`replaceToInsertDoc`),
})
suite.Require().Nil(err, "replaceToInsertDoc get failed")
_, err = testBlkReplace(txn, TransactionReplaceOptions{
Document: replaceToInsertGetRes,
Value: testDummy2,
})
suite.Require().Nil(err, "replaceToInsertDoc replace failed")
txn = suite.serializeUnserializeTxn(txns, txn)
// Cannot insert after serialize.
txn2, err := txns.BeginTransaction(nil)
suite.Require().Nil(err, "txn2 begin failed")
err = txn2.NewAttempt()
suite.Require().Nil(err, "txn2 attempt start failed")
_, err = testBlkInsert(txn2, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`replaceToInsertDoc`),
Value: testDummy3,
})
suite.Assert().Error(err, "replaceToInsertDoc insert from T2 should have failed")
suite.assertStagedDoc("replaceToInsertDoc", jsonMutationReplace, testDummy2, false, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("replaceToInsertDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsReplaceTxn1ReplaceTxn2() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy1 := []byte(`{"name":"joel"}`)
testDummy2 := []byte(`{"name":"mike"}`)
testDummy3 := []byte(`{"name":"frank"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
opHarness.PushOp(agent.Set(SetOptions{
Key: []byte("replaceToReplaceDoc"),
Value: testDummy1,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *StoreResult, err error) {
opHarness.Wrap(func() {
if err != nil {
opHarness.Fatalf("Set command failed: %v", err)
}
})
}))
opHarness.Wait(0)
replaceToReplaceGetRes, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`replaceToReplaceDoc`),
})
suite.Require().Nil(err, "replaceToReplaceDoc get1 failed")
replaceToReplaceReplaceRes, err := testBlkReplace(txn, TransactionReplaceOptions{
Document: replaceToReplaceGetRes,
Value: testDummy2,
})
suite.Require().Nil(err, "replaceToReplaceDoc replace failed")
txn = suite.serializeUnserializeTxn(txns, txn)
replaceToReplaceGet2Res, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`replaceToReplaceDoc`),
})
suite.Require().Nil(err, "replaceToReplaceDoc get2 failed")
suite.Assert().Equal(replaceToReplaceReplaceRes.Cas, replaceToReplaceGet2Res.Cas, "replaceToReplaceDoc replace and get cas did not match")
suite.Assert().Equal(replaceToReplaceReplaceRes.Meta, replaceToReplaceGet2Res.Meta, "replaceToReplaceDoc replace and get meta did not match")
log.Printf("replaceToReplaceDoc get2 result: %+v", replaceToReplaceGet2Res)
replaceToReplaceReplace2Res, err := testBlkReplace(txn, TransactionReplaceOptions{
Document: replaceToReplaceGet2Res,
Value: testDummy3,
})
suite.Require().Nil(err, "replaceToReplaceDoc replace failed")
log.Printf("replaceToReplaceDoc replace result: %+v", replaceToReplaceReplace2Res)
txn2, err := txns.BeginTransaction(nil)
suite.Require().Nil(err, "txn2 begin failed")
err = txn2.NewAttempt()
suite.Require().Nil(err, "txn2 attempt start failed")
_, err = testBlkReplace(txn2, TransactionReplaceOptions{
Document: replaceToReplaceGet2Res,
Value: testDummy1,
})
suite.Assert().Error(err, "replaceToReplaceDoc replace from T2 should have failed")
suite.assertStagedDoc("replaceToReplaceDoc", jsonMutationReplace, testDummy3, false, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("replaceToReplaceDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsReplaceTxn1RemoveTxn2() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy1 := []byte(`{"name":"joel"}`)
testDummy2 := []byte(`{"name":"mike"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
opHarness.PushOp(agent.Set(SetOptions{
Key: []byte("replaceToRemoveDoc"),
Value: testDummy1,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *StoreResult, err error) {
opHarness.Wrap(func() {
if err != nil {
opHarness.Fatalf("Set command failed: %v", err)
}
})
}))
opHarness.Wait(0)
replaceToRemoveGetRes, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`replaceToRemoveDoc`),
})
suite.Require().Nil(err, "replaceToRemoveDoc get1 failed")
replaceToRemoveReplaceRes, err := testBlkReplace(txn, TransactionReplaceOptions{
Document: replaceToRemoveGetRes,
Value: testDummy2,
})
suite.Require().Nil(err, "replaceToRemoveDoc replace failed")
txn = suite.serializeUnserializeTxn(txns, txn)
replaceToRemoveGet2Res, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`replaceToRemoveDoc`),
})
suite.Require().Nil(err, "replaceToRemoveDoc get2 failed")
suite.Assert().Equal(replaceToRemoveReplaceRes.Cas, replaceToRemoveGet2Res.Cas, "replaceToRemoveDoc replace and get cas did not match")
suite.Assert().Equal(replaceToRemoveReplaceRes.Meta, replaceToRemoveGet2Res.Meta, "replaceToRemoveDoc replace and get meta did not match")
log.Printf("replaceToRemoveDoc get2 result: %+v", replaceToRemoveGet2Res)
replaceToRemoveRemoveRes, err := testBlkRemove(txn, TransactionRemoveOptions{
Document: replaceToRemoveGet2Res,
})
suite.Require().Nil(err, "replaceToRemoveDoc remove failed")
log.Printf("replaceToRemoveDoc remove result: %+v", replaceToRemoveRemoveRes)
txn2, err := txns.BeginTransaction(nil)
suite.Require().Nil(err, "txn2 begin failed")
err = txn2.NewAttempt()
suite.Require().Nil(err, "txn2 attempt start failed")
_, err = testBlkRemove(txn2, TransactionRemoveOptions{
Document: replaceToRemoveGet2Res,
})
suite.Assert().Error(err, "replaceToRemoveDoc remove from T2 should have failed")
suite.assertStagedDoc("replaceToRemoveDoc", jsonMutationRemove, []byte{}, false, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("replaceToRemoveDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsRemoveTxn1InsertTxn2() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy1 := []byte(`{"name":"joel"}`)
testDummy3 := []byte(`{"name":"frank"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
opHarness.PushOp(agent.Set(SetOptions{
Key: []byte("removeToInsertDoc"),
Value: testDummy1,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *StoreResult, err error) {
opHarness.Wrap(func() {
if err != nil {
opHarness.Fatalf("Set command failed: %v", err)
}
})
}))
opHarness.Wait(0)
removeToInsertGetRes, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`removeToInsertDoc`),
})
suite.Require().Nil(err, "removeToInsertDoc get failed")
_, err = testBlkRemove(txn, TransactionRemoveOptions{
Document: removeToInsertGetRes,
})
suite.Require().Nil(err, "removeToInsertDoc remove failed")
txn = suite.serializeUnserializeTxn(txns, txn)
_, err = testBlkInsert(txn, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`removeToInsertDoc`),
Value: testDummy3,
})
suite.Require().Nil(err, "removeToInsertDoc insert failed")
txn2, err := txns.BeginTransaction(nil)
suite.Require().Nil(err, "txn2 begin failed")
err = txn2.NewAttempt()
suite.Require().Nil(err, "txn2 attempt start failed")
_, err = testBlkInsert(txn2, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`removeToInsertDoc`),
Value: testDummy1,
})
suite.Assert().Error(err, "removeToInsertDoc insert from T2 should have failed")
suite.assertStagedDoc("removeToInsertDoc", jsonMutationReplace, testDummy3, false, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("removeToInsertDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsRemoveTxn1ReplaceTxn2() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy1 := []byte(`{"name":"joel"}`)
testDummy3 := []byte(`{"name":"frank"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
opHarness.PushOp(agent.Set(SetOptions{
Key: []byte("removeToReplaceDoc"),
Value: testDummy1,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *StoreResult, err error) {
opHarness.Wrap(func() {
if err != nil {
opHarness.Fatalf("Set command failed: %v", err)
}
})
}))
opHarness.Wait(0)
removeToReplaceGetRes, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`removeToReplaceDoc`),
})
suite.Require().Nil(err, "removeToReplaceDoc get failed")
_, err = testBlkRemove(txn, TransactionRemoveOptions{
Document: removeToReplaceGetRes,
})
suite.Require().Nil(err, "removeToReplaceDoc remove failed")
txn = suite.serializeUnserializeTxn(txns, txn)
_, err = testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`removeToReplaceDoc`),
})
suite.Require().NotNil(err, "removeToReplaceDoc get2 should have failed")
txn2, err := txns.BeginTransaction(nil)
suite.Require().Nil(err, "txn2 begin failed")
err = txn2.NewAttempt()
suite.Require().Nil(err, "txn2 attempt start failed")
_, err = testBlkReplace(txn2, TransactionReplaceOptions{
Document: removeToReplaceGetRes,
Value: testDummy3,
})
suite.Assert().Error(err, "removeDoc replace from T2 should have failed")
suite.assertStagedDoc("removeToReplaceDoc", jsonMutationRemove, []byte{}, false, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("removeToReplaceDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsRemoveTxn1RemoveTxn2() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy1 := []byte(`{"name":"joel"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
opHarness.PushOp(agent.Set(SetOptions{
Key: []byte("removeToRemoveDoc"),
Value: testDummy1,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
}, func(result *StoreResult, err error) {
opHarness.Wrap(func() {
if err != nil {
opHarness.Fatalf("Set command failed: %v", err)
}
})
}))
opHarness.Wait(0)
removeToRemoveGetRes, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`removeToRemoveDoc`),
})
suite.Require().Nil(err, "removeToRemoveDoc get failed")
_, err = testBlkRemove(txn, TransactionRemoveOptions{
Document: removeToRemoveGetRes,
})
suite.Require().Nil(err, "removeToRemoveDoc remove failed")
txn = suite.serializeUnserializeTxn(txns, txn)
_, err = testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`removeToRemoveDoc`),
})
suite.Require().NotNil(err, "removeToRemoveDoc get2 should have failed")
suite.assertStagedDoc("removeToRemoveDoc", jsonMutationRemove, []byte{}, false, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("removeToRemoveDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsInsertTxn1InsertTxn2() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy2 := []byte(`{"name":"mike"}`)
testDummy3 := []byte(`{"name":"frank"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
_, err := testBlkInsert(txn, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`insertToInsertDoc`),
Value: testDummy2,
})
suite.Require().Nil(err, "insertToInsertDoc insert failed")
txn = suite.serializeUnserializeTxn(txns, txn)
// No insert here, that'd fail the commit.
txn2, err := txns.BeginTransaction(nil)
suite.Require().Nil(err, "txn2 begin failed")
err = txn2.NewAttempt()
suite.Require().Nil(err, "txn2 attempt start failed")
_, err = testBlkInsert(txn2, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`insertToInsertDoc`),
Value: testDummy3,
})
suite.Require().Error(err, "insertToInsertDoc insert from T2 should have failed")
suite.assertStagedDoc("insertToInsertDoc", jsonMutationInsert, testDummy2, true, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("insertToInsertDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsInsertReplace() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy2 := []byte(`{"name":"mike"}`)
testDummy3 := []byte(`{"name":"frank"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
insertToReplaceInsertRes, err := testBlkInsert(txn, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`insertToReplaceDoc`),
Value: testDummy2,
})
suite.Require().Nil(err, "insertToReplaceDoc insert failed")
txn = suite.serializeUnserializeTxn(txns, txn)
insertToReplaceGet2Res, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`insertToReplaceDoc`),
})
suite.Require().Nil(err, "insertToReplaceDoc get2 failed")
suite.Assert().Equal(insertToReplaceInsertRes.Cas, insertToReplaceGet2Res.Cas, "insertToReplaceDoc insert and get cas did not match")
suite.Assert().Equal(insertToReplaceInsertRes.Meta, insertToReplaceGet2Res.Meta, "insertToReplaceDoc insert and get meta did not match")
log.Printf("insertToReplaceDoc get2 result: %+v", insertToReplaceGet2Res)
insertToReplaceReplaceRes, err := testBlkReplace(txn, TransactionReplaceOptions{
Document: insertToReplaceInsertRes,
Value: testDummy3,
})
suite.Require().Nil(err, "insertToReplaceDoc replace failed")
log.Printf("insertToReplaceDoc replace result: %+v", insertToReplaceReplaceRes)
// Impossible to have a txn2 with a replace.
suite.assertStagedDoc("insertToReplaceDoc", jsonMutationInsert, testDummy3, true, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged("insertToReplaceDoc", agent, opHarness)
}
func (suite *StandardTestSuite) TestTransactionsInsertRemove() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy2 := []byte(`{"name":"mike"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
insertToRemoveInsertRes, err := testBlkInsert(txn, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`insertToRemoveDoc`),
Value: testDummy2,
})
suite.Require().Nil(err, "insertToRemoveDoc insert failed")
txn = suite.serializeUnserializeTxn(txns, txn)
insertToRemoveGet2Res, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(`insertToRemoveDoc`),
})
suite.Require().Nil(err, "insertToRemoveDoc get2 failed")
suite.Assert().Equal(insertToRemoveInsertRes.Cas, insertToRemoveGet2Res.Cas, "insertToRemoveDoc insert and get cas did not match")
suite.Assert().Equal(insertToRemoveInsertRes.Meta, insertToRemoveGet2Res.Meta, "insertToRemoveDoc insert and get meta did not match")
_, err = testBlkRemove(txn, TransactionRemoveOptions{
Document: insertToRemoveGet2Res,
})
suite.Require().Nil(err, "insertToRemoveDoc remove failed")
// Impossible to have a txn2 with a remove.
suite.assertStagedDoc("insertToRemoveDoc", "", nil, true, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertStagedDoc("insertToRemoveDoc", "", nil, true, agent, opHarness)
}
// This test ensures that the addLostCleanupLocation function is registered on an attempt
// after serialization/deserialization of the transaction.
func (suite *StandardTestSuite) TestTransactionsInsertRemoveEarlySerialize() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy2 := []byte(`{"name":"mike"}`)
txns, txn := suite.initTransactionAndAttempt(agent)
// We do this before any ops so that we trigger addLostCleanupLocation after setting the ATR to pending.
txn = suite.serializeUnserializeTxn(txns, txn)
docId := []byte(uuid.NewString())
insertToRemoveInsertRes, err := testBlkInsert(txn, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: docId,
Value: testDummy2,
})
suite.Require().Nil(err, "Early serialization insert failed")
insertToRemoveGet2Res, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: docId,
})
suite.Require().Nil(err, "Early serialization get2 failed")
suite.Assert().Equal(insertToRemoveInsertRes.Cas, insertToRemoveGet2Res.Cas, "Early serialization insert and get cas did not match")
suite.Assert().Equal(insertToRemoveInsertRes.Meta, insertToRemoveGet2Res.Meta, "Early serialization insert and get meta did not match")
_, err = testBlkRemove(txn, TransactionRemoveOptions{
Document: insertToRemoveGet2Res,
})
suite.Require().Nil(err, "Early serialization remove failed")
// Impossible to have a txn2 with a remove.
suite.assertStagedDoc(string(docId), "", nil, true, agent, opHarness)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertStagedDoc(string(docId), "", nil, true, agent, opHarness)
}
func (suite *StandardTestSuite) serializeUnserializeTxn(txns *TransactionsManager, txn *Transaction) *Transaction {
txnBytes, err := testBlkSerialize(txn)
suite.Require().Nil(err, "txn serialize failed")
txn, err = txns.ResumeTransactionAttempt(txnBytes, nil)
suite.Require().Nil(err, err, "txn resume failed")
return txn
}
func (suite *StandardTestSuite) TestTransactionsLogger() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
agent, opHarness := suite.GetAgentAndHarness()
testDummy2 := []byte(`{"name":"mike"}`)
txns, err := InitTransactions(&TransactionsConfig{
DurabilityLevel: TransactionDurabilityLevelNone,
BucketAgentProvider: func(bucketName string) (*Agent, string, error) {
// We can always return just this one agent as we only actually
// use a single bucket for this entire test.
return agent, "", nil
},
ExpirationTime: 60 * time.Second,
})
suite.Require().Nil(err, err)
logger := NewInMemoryTransactionLogger()
txn, err := txns.BeginTransaction(&TransactionOptions{
TransactionLogger: logger,
})
suite.Require().Nil(err, err)
// Start the attempt
err = txn.NewAttempt()
suite.Require().Nil(err, err)
key := uuid.NewString()
_, err = testBlkInsert(txn, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(key),
Value: testDummy2,
})
suite.Require().Nil(err, "insert failed")
_, err = testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(key),
})
suite.Require().Nil(err, "get failed")
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
suite.assertDocNotStaged(key, agent, opHarness)
entries := logger.Logs()
for _, entry := range entries {
suite.Assert().NotZero(entry.Level)
items := strings.SplitN(entry.String(), " ", 3)
if suite.Assert().Len(items, 3) {
suite.Assert().Len(items[0], 12)
suite.Assert().Equal(txn.ID()[:5]+"/"+txn.Attempt().ID[:5], items[1])
suite.Assert().NotEmpty(items[2])
}
}
}
func (suite *StandardTestSuite) TestTransactionsInsertGetReplaceRemoveCommitUnits() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
suite.EnsureSupportsFeature(TestFeatureResourceUnits)
agent, _ := suite.GetAgentAndHarness()
val1 := []byte(`{"name":"mike"}`)
val2 := []byte(`{"name":"dave"}`)
txns, err := InitTransactions(&TransactionsConfig{
DurabilityLevel: TransactionDurabilityLevelNone,
BucketAgentProvider: func(bucketName string) (*Agent, string, error) {
// We can always return just this one agent as we only actually
// use a single bucket for this entire test.
return agent, "", nil
},
ExpirationTime: 60 * time.Second,
})
suite.Require().Nil(err, err)
opts := &TransactionOptions{}
var readUnits uint32
var writeUnits uint32
var numOps uint32
opts.Internal.ResourceUnitCallback = func(result *ResourceUnitResult) {
readUnits += uint32(result.ReadUnits)
writeUnits += uint32(result.WriteUnits)
numOps++
}
txn, err := txns.BeginTransaction(opts)
suite.Require().Nil(err, err)
// Start the attempt
err = txn.NewAttempt()
suite.Require().Nil(err, err)
key := uuid.NewString()
_, err = testBlkInsert(txn, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(key),
Value: val1,
})
suite.Require().Nil(err, "insert failed")
// We expect there to be 2 writes - the atr and the doc itself.
suite.Assert().Equal(uint32(2), writeUnits)
suite.Assert().Equal(uint32(0), readUnits)
suite.Assert().Equal(uint32(2), numOps)
getRes, err := testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(key),
})
suite.Require().Nil(err, "get failed")
// Get should add a read - of the doc.
suite.Assert().Equal(uint32(2), writeUnits)
suite.Assert().Equal(uint32(1), readUnits)
suite.Assert().Equal(uint32(3), numOps)
_, err = testBlkReplace(txn, TransactionReplaceOptions{
Document: getRes,
Value: val2,
})
suite.Require().Nil(err, "replace failed")
// Replace should add a write and a read (as one op) - of the doc.
suite.Assert().Equal(uint32(3), writeUnits)
suite.Assert().Equal(uint32(2), readUnits)
suite.Assert().Equal(uint32(4), numOps)
getRes, err = testBlkGet(txn, TransactionGetOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(key),
})
suite.Require().Nil(err, "get failed")
// Get should add a read - of the doc.
suite.Assert().Equal(uint32(3), writeUnits)
suite.Assert().Equal(uint32(3), readUnits)
suite.Assert().Equal(uint32(5), numOps)
_, err = testBlkRemove(txn, TransactionRemoveOptions{
Document: getRes,
})
suite.Require().Nil(err, "remove failed")
// Remove should add a write and a read (as one op) - of the doc.
suite.Assert().Equal(uint32(4), writeUnits)
suite.Assert().Equal(uint32(4), readUnits)
suite.Assert().Equal(uint32(6), numOps)
err = testBlkCommit(txn)
suite.Require().Nil(err, "commit failed")
// Commit should add a write and a read (as one op) - of the atr and the doc.
suite.Assert().Equal(uint32(6), writeUnits)
suite.Assert().Equal(uint32(6), readUnits)
suite.Assert().Equal(uint32(8), numOps)
}
func (suite *StandardTestSuite) TestTransactionsInsertRollbacktUnits() {
suite.EnsureSupportsFeature(TestFeatureTransactions)
suite.EnsureSupportsFeature(TestFeatureResourceUnits)
agent, _ := suite.GetAgentAndHarness()
val1 := []byte(`{"name":"mike"}`)
txns, err := InitTransactions(&TransactionsConfig{
DurabilityLevel: TransactionDurabilityLevelNone,
BucketAgentProvider: func(bucketName string) (*Agent, string, error) {
// We can always return just this one agent as we only actually
// use a single bucket for this entire test.
return agent, "", nil
},
ExpirationTime: 60 * time.Second,
})
suite.Require().Nil(err, err)
opts := &TransactionOptions{}
var readUnits uint32
var writeUnits uint32
var numOps uint32
opts.Internal.ResourceUnitCallback = func(result *ResourceUnitResult) {
readUnits += uint32(result.ReadUnits)
writeUnits += uint32(result.WriteUnits)
numOps++
}
txn, err := txns.BeginTransaction(opts)
suite.Require().Nil(err, err)
// Start the attempt
err = txn.NewAttempt()
suite.Require().Nil(err, err)
key := uuid.NewString()
_, err = testBlkInsert(txn, TransactionInsertOptions{
Agent: agent,
ScopeName: suite.ScopeName,
CollectionName: suite.CollectionName,
Key: []byte(key),
Value: val1,
})
suite.Require().Nil(err, "insert failed")
// We expect there to be 2 writes - the atr and the doc itself.
suite.Assert().Equal(uint32(2), writeUnits)
suite.Assert().Equal(uint32(0), readUnits)
suite.Assert().Equal(uint32(2), numOps)
err = testBlkRollback(txn)
suite.Require().Nil(err, "rollback failed")
// Rollback should add 3 writes and 3 reads - there are 2 requests against the atr and one against the doc.
suite.Assert().Equal(uint32(5), writeUnits)
suite.Assert().Equal(uint32(3), readUnits)
suite.Assert().Equal(uint32(5), numOps)
}
gocbcore-10.2.3/transactions_utils.go 0000664 0000000 0000000 00000005046 14417540156 0017653 0 ustar 00root root 0000000 0000000 // Copyright 2021 Couchbase
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gocbcore
import (
"time"
"github.com/couchbase/gocbcore/v10/memd"
)
func transactionsDurabilityLevelToMemd(durabilityLevel TransactionDurabilityLevel) memd.DurabilityLevel {
switch durabilityLevel {
case TransactionDurabilityLevelNone:
return memd.DurabilityLevel(0)
case TransactionDurabilityLevelMajority:
return memd.DurabilityLevelMajority
case TransactionDurabilityLevelMajorityAndPersistToActive:
return memd.DurabilityLevelMajorityAndPersistOnMaster
case TransactionDurabilityLevelPersistToMajority:
return memd.DurabilityLevelPersistToMajority
case TransactionDurabilityLevelUnknown:
panic("unexpected unset durability level")
default:
panic("unexpected durability level")
}
}
func transactionsDurabilityLevelToShorthand(durabilityLevel TransactionDurabilityLevel) string {
switch durabilityLevel {
case TransactionDurabilityLevelNone:
return "n"
case TransactionDurabilityLevelMajority:
return "m"
case TransactionDurabilityLevelMajorityAndPersistToActive:
return "pa"
case TransactionDurabilityLevelPersistToMajority:
return "pm"
default:
// If it's an unknown durability level, default to majority.
return "m"
}
}
func transactionsDurabilityLevelFromShorthand(durabilityLevel string) TransactionDurabilityLevel {
switch durabilityLevel {
case "m":
return TransactionDurabilityLevelMajority
case "pa":
return TransactionDurabilityLevelMajorityAndPersistToActive
case "pm":
return TransactionDurabilityLevelPersistToMajority
default:
// If there is no durability level present or it's set to none then we'll set to majority.
return TransactionDurabilityLevelMajority
}
}
func transactionsMutationTimeouts(opTimeout time.Duration, durability TransactionDurabilityLevel) (time.Time, time.Duration) {
var deadline time.Time
var duraTimeout time.Duration
if opTimeout > 0 {
deadline = time.Now().Add(opTimeout)
if durability > TransactionDurabilityLevelNone {
duraTimeout = opTimeout
}
}
return deadline, duraTimeout
}
gocbcore-10.2.3/util.go 0000664 0000000 0000000 00000002645 14417540156 0014702 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"crypto/rand"
"encoding/json"
"fmt"
"strings"
)
func getMapValueString(dict map[string]interface{}, key string, def string) string {
if dict != nil {
if val, ok := dict[key]; ok {
if valStr, ok := val.(string); ok {
return valStr
}
}
}
return def
}
func getMapValueBool(dict map[string]interface{}, key string, def bool) bool {
if dict != nil {
if val, ok := dict[key]; ok {
if valStr, ok := val.(bool); ok {
return valStr
}
}
}
return def
}
func randomCbUID() []byte {
out := make([]byte, 8)
_, err := rand.Read(out)
if err != nil {
logWarnf("Crypto read failed: %s", err)
}
return out
}
func formatCbUID(data []byte) string {
return fmt.Sprintf("%02x%02x%02x%02x%02x%02x%02x%02x",
data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7])
}
func clientInfoString(connID, userAgent string) string {
agentName := "gocbcore/" + goCbCoreVersionStr
if userAgent != "" {
agentName += " " + userAgent
}
clientInfo := struct {
Agent string `json:"a"`
ConnID string `json:"i"`
}{
Agent: agentName,
ConnID: connID,
}
clientInfoBytes, err := json.Marshal(clientInfo)
if err != nil {
logDebugf("Failed to generate client info string: %s", err)
}
return string(clientInfoBytes)
}
func trimSchemePrefix(address string) string {
idx := strings.Index(address, "://")
if idx < 0 {
return address
}
return address[idx+len("://"):]
}
gocbcore-10.2.3/util_test.go 0000664 0000000 0000000 00000001254 14417540156 0015734 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestTrimSchemePrefix(t *testing.T) {
type test struct {
name, address string
}
tests := []*test{
{
name: "None",
address: "hostname:8091",
},
{
name: "HTTP",
address: "http://hostname:8091",
},
{
name: "HTTPS",
address: "https://hostname:8091",
},
{
name: "Couchbase",
address: "couchbase://hostname:8091",
},
{
name: "Couchbases",
address: "couchbases://hostname:8091",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
require.Equal(t, "hostname:8091", trimSchemePrefix(test.address))
})
}
}
gocbcore-10.2.3/vbucketmap.go 0000664 0000000 0000000 00000003440 14417540156 0016060 0 ustar 00root root 0000000 0000000 package gocbcore
type vbucketMap struct {
entries [][]int
numReplicas int
}
func newVbucketMap(entries [][]int, numReplicas int) *vbucketMap {
vbMap := vbucketMap{
entries: entries,
numReplicas: numReplicas,
}
return &vbMap
}
func (vbMap vbucketMap) IsValid() bool {
return len(vbMap.entries) > 0 && len(vbMap.entries[0]) > 0
}
func (vbMap vbucketMap) NumVbuckets() int {
return len(vbMap.entries)
}
func (vbMap vbucketMap) NumReplicas() int {
return vbMap.numReplicas
}
func (vbMap vbucketMap) VbucketByKey(key []byte) uint16 {
return uint16(cbCrc(key) % uint32(len(vbMap.entries)))
}
func (vbMap vbucketMap) NodeByVbucket(vbID uint16, replicaID uint32) (int, error) {
if vbID >= uint16(len(vbMap.entries)) {
return 0, errInvalidVBucket
}
if replicaID >= uint32(len(vbMap.entries[vbID])) {
return 0, errInvalidReplica
}
return vbMap.entries[vbID][replicaID], nil
}
func (vbMap vbucketMap) VbucketsOnServer(index int) ([]uint16, error) {
vbList, err := vbMap.VbucketsByServer(0)
if err != nil {
return nil, err
}
if len(vbList) <= index {
// Invalid server index
return nil, errInvalidReplica
}
return vbList[index], nil
}
func (vbMap vbucketMap) VbucketsByServer(replicaID int) ([][]uint16, error) {
var vbList [][]uint16
// We do not currently support listing for all replicas at once
if replicaID < 0 {
return nil, errInvalidReplica
}
for vbID, entry := range vbMap.entries {
if len(entry) <= replicaID {
continue
}
serverID := entry[replicaID]
for len(vbList) <= serverID {
vbList = append(vbList, nil)
}
vbList[serverID] = append(vbList[serverID], uint16(vbID))
}
return vbList, nil
}
func (vbMap vbucketMap) NodeByKey(key []byte, replicaID uint32) (int, error) {
return vbMap.NodeByVbucket(vbMap.VbucketByKey(key), replicaID)
}
gocbcore-10.2.3/version.go 0000664 0000000 0000000 00000000216 14417540156 0015402 0 ustar 00root root 0000000 0000000 package gocbcore
// Version returns a string representation of the current SDK version.
func Version() string {
return goCbCoreVersionStr
}
gocbcore-10.2.3/viewscomponent.go 0000664 0000000 0000000 00000012153 14417540156 0017000 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/url"
"strings"
"time"
)
// ViewQueryRowReader providers access to the rows of a view query
type ViewQueryRowReader struct {
streamer *queryStreamer
}
// NextRow reads the next rows bytes from the stream
func (q *ViewQueryRowReader) NextRow() []byte {
return q.streamer.NextRow()
}
// Err returns any errors that occurred during streaming.
func (q ViewQueryRowReader) Err() error {
return q.streamer.Err()
}
// MetaData fetches the non-row bytes streamed in the response.
func (q *ViewQueryRowReader) MetaData() ([]byte, error) {
return q.streamer.MetaData()
}
// Close immediately shuts down the connection
func (q *ViewQueryRowReader) Close() error {
return q.streamer.Close()
}
// ViewQueryOptions represents the various options available for a view query.
type ViewQueryOptions struct {
DesignDocumentName string
ViewType string
ViewName string
Options url.Values
RetryStrategy RetryStrategy
Deadline time.Time
// Internal: This should never be used and is not supported.
User string
TraceContext RequestSpanContext
}
func wrapViewQueryError(req *httpRequest, ddoc, view string, err error, errBody string, statusCode int) *ViewError {
if err == nil {
err = errors.New("view error")
}
ierr := &ViewError{
InnerError: err,
}
if req != nil {
ierr.Endpoint = req.Endpoint
ierr.RetryAttempts = req.RetryAttempts()
ierr.RetryReasons = req.RetryReasons()
}
ierr.ErrorText = errBody
ierr.HTTPResponseCode = statusCode
ierr.DesignDocumentName = ddoc
ierr.ViewName = view
return ierr
}
func parseViewQueryError(req *httpRequest, ddoc, view string, resp *HTTPResponse) *ViewError {
var err error
var errorDescs []ViewQueryErrorDesc
respBody, readErr := ioutil.ReadAll(resp.Body)
if readErr == nil {
var errsMap map[string]string
var errsArr []string
if err := json.Unmarshal(respBody, &errsArr); err == nil {
errorDescs = make([]ViewQueryErrorDesc, len(errsArr))
for errIdx, errMessage := range errsArr {
errorDescs[errIdx] = ViewQueryErrorDesc{
SourceNode: "",
Message: errMessage,
}
}
} else if err := json.Unmarshal(respBody, &errsMap); err == nil {
for errNode, errMessage := range errsMap {
errorDescs = append(errorDescs, ViewQueryErrorDesc{
SourceNode: errNode,
Message: errMessage,
})
}
}
}
if resp.StatusCode == 401 {
err = errAuthenticationFailure
} else if resp.StatusCode == 404 {
err = errViewNotFound
}
if len(errorDescs) >= 1 {
firstErrMsg := errorDescs[0].Message
if strings.Contains(firstErrMsg, "not_found") {
err = errViewNotFound
}
}
var errText string
if err == nil {
errText = string(respBody)
}
errOut := wrapViewQueryError(req, ddoc, view, err, errText, resp.StatusCode)
errOut.Errors = errorDescs
return errOut
}
type viewQueryComponent struct {
httpComponent *httpComponent
tracer *tracerComponent
}
func newViewQueryComponent(httpComponent *httpComponent, tracer *tracerComponent) *viewQueryComponent {
return &viewQueryComponent{
httpComponent: httpComponent,
tracer: tracer,
}
}
// ViewQuery executes a view query
func (vqc *viewQueryComponent) ViewQuery(opts ViewQueryOptions, cb ViewQueryCallback) (PendingOp, error) {
tracer := vqc.tracer.StartTelemeteryHandler(metricValueServiceViewsValue, "ViewQuery", opts.TraceContext)
reqURI := fmt.Sprintf("/_design/%s/%s/%s?%s",
opts.DesignDocumentName, opts.ViewType, opts.ViewName, opts.Options.Encode())
ctx, cancel := context.WithCancel(context.Background())
ireq := &httpRequest{
Service: CapiService,
Method: "GET",
Path: reqURI,
IsIdempotent: true,
Deadline: opts.Deadline,
RetryStrategy: opts.RetryStrategy,
RootTraceContext: tracer.RootContext(),
Context: ctx,
CancelFunc: cancel,
User: opts.User,
}
ddoc := opts.DesignDocumentName
view := opts.ViewName
go func() {
res, err := vqc.viewQuery(ireq, ddoc, view)
if err != nil {
cancel()
tracer.Finish()
cb(nil, err)
return
}
tracer.Finish()
cb(res, nil)
}()
return ireq, nil
}
func (vqc *viewQueryComponent) viewQuery(ireq *httpRequest, ddoc, view string) (*ViewQueryRowReader, error) {
resp, err := vqc.httpComponent.DoInternalHTTPRequest(ireq, false)
if err != nil {
if errors.Is(err, ErrRequestCanceled) {
return nil, err
}
// execHTTPRequest will handle retrying due to in-flight socket close based
// on whether or not IsIdempotent is set on the httpRequest
return nil, wrapViewQueryError(ireq, ddoc, view, err, "", 0)
}
if resp.StatusCode != 200 {
viewErr := parseViewQueryError(ireq, ddoc, view, resp)
// viewErr is already wrapped here
return nil, viewErr
}
streamer, err := newQueryStreamer(resp.Body, "rows")
if err != nil {
respBody, readErr := ioutil.ReadAll(resp.Body)
if readErr != nil {
logDebugf("Failed to read response body: %v", readErr)
}
return nil, wrapViewQueryError(ireq, ddoc, view, err, string(respBody), resp.StatusCode)
}
return &ViewQueryRowReader{
streamer: streamer,
}, nil
}
gocbcore-10.2.3/zombielogger_component.go 0000664 0000000 0000000 00000011237 14417540156 0020471 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"fmt"
"sort"
"sync"
"time"
)
type zombieLogEntry struct {
connectionID string
operationID string
remoteSocket string
localSocket string
duration time.Duration
operationName string
}
type zombieLogItem struct {
ConnectionID string `json:"last_local_id"`
OperationID string `json:"operation_id"`
RemoteSocket string `json:"last_remote_socket,omitempty"`
LocalSocket string `json:"last_local_socket,omitempty"`
ServerDurationUs uint64 `json:"last_server_duration_us,omitempty"`
OperationName string `json:"operation_name"`
}
type zombieLogJsonEntry struct {
Count int `json:"total_count"`
Top []zombieLogItem `json:"top_requests"`
}
type zombieLogService map[string]zombieLogJsonEntry
type zombieLoggerComponent struct {
zombieLock sync.RWMutex
zombieOps []*zombieLogEntry
interval time.Duration
sampleSize int
stopSig chan struct{}
}
func newZombieLoggerComponent(interval time.Duration, sampleSize int) *zombieLoggerComponent {
return &zombieLoggerComponent{
// zombieOps must have a static capacity for its lifetime, the capacity should
// never be altered so that it is consistent across the zombieLogger and
// recordZombieResponse.
zombieOps: make([]*zombieLogEntry, 0, sampleSize),
interval: interval,
sampleSize: sampleSize,
stopSig: make(chan struct{}),
}
}
func (zlc *zombieLoggerComponent) Start() {
lastTick := time.Now()
for {
select {
case <-zlc.stopSig:
return
case <-time.After(zlc.interval):
}
lastTick = lastTick.Add(zlc.interval)
jsonBytes := zlc.createOutput()
if len(jsonBytes) == 0 {
continue
}
logWarnf("Orphaned responses observed:\n %s", jsonBytes)
}
}
func (zlc *zombieLoggerComponent) createOutput() []byte {
// Preallocate space to copy the ops into...
oldOps := make([]*zombieLogEntry, zlc.sampleSize)
zlc.zombieLock.Lock()
// Escape early if we have no ops to log...
if len(zlc.zombieOps) == 0 {
zlc.zombieLock.Unlock()
return nil
}
// Copy out our ops so we can cheaply print them out without blocking
// our ops from actually being recorded in other goroutines (which would
// effectively slow down the op pipeline for logging).
oldOps = oldOps[0:len(zlc.zombieOps)]
copy(oldOps, zlc.zombieOps)
zlc.zombieOps = zlc.zombieOps[:0]
zlc.zombieLock.Unlock()
entries := zombieLogJsonEntry{
Top: make([]zombieLogItem, len(oldOps)),
}
for i := 0; i < len(oldOps); i++ {
op := oldOps[i]
entries.Top[len(oldOps)-i-1] = zombieLogItem{
OperationID: op.operationID,
ConnectionID: op.connectionID,
RemoteSocket: op.remoteSocket,
LocalSocket: op.localSocket,
ServerDurationUs: uint64(op.duration.Microseconds()),
OperationName: op.operationName,
}
}
entries.Count = len(entries.Top)
jsonBytes, err := json.Marshal(zombieLogService{
"kv": entries,
})
if err != nil {
logDebugf("Failed to generate zombie logging JSON: %s", err)
}
return jsonBytes
}
func (zlc *zombieLoggerComponent) Stop() {
close(zlc.stopSig)
}
func (zlc *zombieLoggerComponent) RecordZombieResponse(resp *memdQResponse, connID, localAddr, remoteAddr string) {
entry := &zombieLogEntry{
connectionID: connID,
operationID: fmt.Sprintf("0x%x", resp.Opaque),
remoteSocket: remoteAddr,
duration: 0,
operationName: resp.Command.Name(),
localSocket: localAddr,
}
if resp.Packet.ServerDurationFrame != nil {
entry.duration = resp.Packet.ServerDurationFrame.ServerDuration
}
zlc.zombieLock.RLock()
if cap(zlc.zombieOps) == 0 || (len(zlc.zombieOps) == cap(zlc.zombieOps) &&
entry.duration < zlc.zombieOps[0].duration) {
// we are at capacity and we are faster than the fastest slow op or somehow in a state where capacity is 0.
zlc.zombieLock.RUnlock()
return
}
zlc.zombieLock.RUnlock()
zlc.zombieLock.Lock()
if cap(zlc.zombieOps) == 0 || (len(zlc.zombieOps) == cap(zlc.zombieOps) &&
entry.duration < zlc.zombieOps[0].duration) {
// we are at capacity and we are faster than the fastest slow op or somehow in a state where capacity is 0.
zlc.zombieLock.Unlock()
return
}
l := len(zlc.zombieOps)
i := sort.Search(l, func(i int) bool { return entry.duration < zlc.zombieOps[i].duration })
// i represents the slot where it should be inserted
if len(zlc.zombieOps) < cap(zlc.zombieOps) {
if i == l {
zlc.zombieOps = append(zlc.zombieOps, entry)
} else {
zlc.zombieOps = append(zlc.zombieOps, nil)
copy(zlc.zombieOps[i+1:], zlc.zombieOps[i:])
zlc.zombieOps[i] = entry
}
} else {
if i == 0 {
zlc.zombieOps[i] = entry
} else {
copy(zlc.zombieOps[0:i-1], zlc.zombieOps[1:i])
zlc.zombieOps[i-1] = entry
}
}
zlc.zombieLock.Unlock()
}
gocbcore-10.2.3/zombielogger_component_test.go 0000664 0000000 0000000 00000007737 14417540156 0021542 0 ustar 00root root 0000000 0000000 package gocbcore
import (
"encoding/json"
"fmt"
"github.com/couchbase/gocbcore/v10/memd"
"time"
)
func (suite *UnitTestSuite) TestZombieLoggerComponent() {
responses := []*memdQResponse{
{
Packet: &memd.Packet{
Command: memd.CmdReplace,
Opaque: 23,
ServerDurationFrame: &memd.ServerDurationFrame{
ServerDuration: 2100 * time.Microsecond,
},
},
sourceAddr: "10.112.210.101",
sourceConnID: "9a1e99041b33322b/54cf79f08d852738",
},
{
Packet: &memd.Packet{
Command: memd.CmdReplace,
Opaque: 24,
ServerDurationFrame: &memd.ServerDurationFrame{
ServerDuration: 2200 * time.Microsecond,
},
},
sourceAddr: "10.112.210.101",
sourceConnID: "9a1e99041b33322b/54cf79f08d852738",
},
{
Packet: &memd.Packet{
Command: memd.CmdReplace,
Opaque: 25,
ServerDurationFrame: &memd.ServerDurationFrame{
ServerDuration: 1100 * time.Microsecond,
},
},
sourceAddr: "10.112.210.101",
sourceConnID: "9a1e99041b33322b/54cf79f08d852738",
},
{
Packet: &memd.Packet{
Command: memd.CmdGet,
Opaque: 27,
ServerDurationFrame: &memd.ServerDurationFrame{
ServerDuration: 2800 * time.Microsecond,
},
},
sourceAddr: "10.112.210.101",
sourceConnID: "9a1e99041b33322b/54cf79f08d852738",
},
{
Packet: &memd.Packet{
Command: memd.CmdReplace,
Opaque: 29,
ServerDurationFrame: &memd.ServerDurationFrame{
ServerDuration: 5000 * time.Microsecond,
},
},
sourceAddr: "10.112.210.101",
sourceConnID: "9a1e99041b33322b/54cf79f08d852738",
},
}
z := newZombieLoggerComponent(1*time.Second, 4)
go z.Start()
for _, r := range responses {
z.RecordZombieResponse(r, "9a1e99041b33322b/54cf79f08d852738", "10.112.210.1", "10.112.210.101")
}
z.Stop()
jsonOutput := z.createOutput()
type expectedOutputFormat struct {
ConnectionID string `json:"last_local_id"`
OperationID string `json:"operation_id"`
RemoteSocket string `json:"last_remote_socket,omitempty"`
LocalSocket string `json:"last_local_socket,omitempty"`
ServerDurationUs uint64 `json:"last_server_duration_us,omitempty"`
OperationName string `json:"operation_name"`
}
expectedOutput := []expectedOutputFormat{
{
ConnectionID: "9a1e99041b33322b/54cf79f08d852738",
OperationID: "0x1d",
RemoteSocket: "10.112.210.101",
LocalSocket: "10.112.210.1",
ServerDurationUs: 5000,
OperationName: memd.CmdReplace.Name(),
},
{
ConnectionID: "9a1e99041b33322b/54cf79f08d852738",
OperationID: "0x1b",
RemoteSocket: "10.112.210.101",
LocalSocket: "10.112.210.1",
ServerDurationUs: 2800,
OperationName: memd.CmdGet.Name(),
},
{
ConnectionID: "9a1e99041b33322b/54cf79f08d852738",
OperationID: "0x18",
RemoteSocket: "10.112.210.101",
LocalSocket: "10.112.210.1",
ServerDurationUs: 2200,
OperationName: memd.CmdReplace.Name(),
},
{
ConnectionID: "9a1e99041b33322b/54cf79f08d852738",
OperationID: "0x17",
RemoteSocket: "10.112.210.101",
LocalSocket: "10.112.210.1",
ServerDurationUs: 2100,
OperationName: memd.CmdReplace.Name(),
},
}
expectedJsonOutput, err := json.Marshal(expectedOutput)
suite.Require().Nil(err)
var mapTopOutput map[string]json.RawMessage
suite.Require().Nil(json.Unmarshal(jsonOutput, &mapTopOutput))
suite.Require().Contains(mapTopOutput, "kv")
var mapInnerOutput map[string]json.RawMessage
suite.Require().Nil(json.Unmarshal(mapTopOutput["kv"], &mapInnerOutput))
suite.Require().Contains(mapInnerOutput, "total_count")
suite.Require().Contains(mapInnerOutput, "top_requests")
var totalCount int
suite.Require().Nil(json.Unmarshal(mapInnerOutput["total_count"], &totalCount))
suite.Assert().Equal(4, totalCount)
suite.Assert().Equal(expectedJsonOutput, []byte(mapInnerOutput["top_requests"]), fmt.Sprintf("Expected output to be %s but was %s", string(expectedJsonOutput), string(mapInnerOutput["top_requests"])))
}