From 7fe55152ec977550b8aac492600d45923b75ac01 Mon Sep 17 00:00:00 2001 From: Rodrigo Gryzinski Date: Tue, 10 Aug 2021 17:07:08 +0100 Subject: [PATCH] Initial commit --- .github/CODEOWNERS | 1 + .github/ISSUE_TEMPLATE/bug_report.md | 31 +++ .github/ISSUE_TEMPLATE/config.yml | 1 + .github/ISSUE_TEMPLATE/feature_request.md | 27 +++ .github/PULL_REQUEST_TEMPLATE.md | 12 ++ .github/workflows/audit.yml | 22 ++ .github/workflows/ci.yml | 130 ++++++++++++ .gitignore | 2 + CHANGELOG.md | 7 + CODE_OF_CONDUCT.md | 3 + CONTRIBUTING.md | 68 ++++++ Cargo.toml | 6 + LICENSE-APACHE | 176 +++++++++++++++ LICENSE-MIT | 21 ++ README.md | 65 ++++++ reqwest-middleware/Cargo.toml | 25 +++ reqwest-middleware/src/client.rs | 248 ++++++++++++++++++++++ reqwest-middleware/src/error.rs | 22 ++ reqwest-middleware/src/lib.rs | 51 +++++ reqwest-middleware/src/middleware.rs | 100 +++++++++ reqwest-retry/.gitignore | 2 + reqwest-retry/CHANGELOG.md | 7 + reqwest-retry/Cargo.toml | 29 +++ reqwest-retry/README.md | 41 ++++ reqwest-retry/src/lib.rs | 7 + reqwest-retry/src/middleware.rs | 139 ++++++++++++ reqwest-retry/src/retryable.rs | 68 ++++++ reqwest-retry/tests/retry.rs | 215 +++++++++++++++++++ reqwest-tracing/CHANGELOG.md | 7 + reqwest-tracing/Cargo.toml | 35 +++ reqwest-tracing/README.md | 71 +++++++ reqwest-tracing/src/lib.rs | 9 + reqwest-tracing/src/middleware.rs | 129 +++++++++++ reqwest-tracing/src/otel.rs | 62 ++++++ 34 files changed, 1839 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/audit.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 Cargo.toml create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 README.md create mode 100644 reqwest-middleware/Cargo.toml create mode 100644 reqwest-middleware/src/client.rs create mode 100644 reqwest-middleware/src/error.rs create mode 100644 reqwest-middleware/src/lib.rs create mode 100644 reqwest-middleware/src/middleware.rs create mode 100644 reqwest-retry/.gitignore create mode 100644 reqwest-retry/CHANGELOG.md create mode 100644 reqwest-retry/Cargo.toml create mode 100644 reqwest-retry/README.md create mode 100644 reqwest-retry/src/lib.rs create mode 100644 reqwest-retry/src/middleware.rs create mode 100644 reqwest-retry/src/retryable.rs create mode 100644 reqwest-retry/tests/retry.rs create mode 100644 reqwest-tracing/CHANGELOG.md create mode 100644 reqwest-tracing/Cargo.toml create mode 100644 reqwest-tracing/README.md create mode 100644 reqwest-tracing/src/lib.rs create mode 100644 reqwest-tracing/src/middleware.rs create mode 100644 reqwest-tracing/src/otel.rs diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..f8ac742 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @TrueLayer/rust-oss diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..25b4792 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +## Bug description + + + +## To Reproduce + + + +## Expected behavior + + + +## Environment + + + +- OS: [e.g. Windows] +- Rust version [e.g. 1.51.0] + +## Additional context + + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..0086358 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..d13231a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,27 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +## Motivations + + + +## Solution + + + +## Alternatives + + + +## Additional context + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..e34116e --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,12 @@ + + + diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 0000000..9e603ac --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,22 @@ +name: Security audit + +on: + schedule: + # Runs at 00:00 UTC everyday + - cron: '0 0 * * *' + push: + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + pull_request: + +jobs: + audit: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - uses: actions-rs/audit-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..80639f7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,130 @@ +on: + push: + branches: [main] + pull_request: +name: CI # Continuous Integration + +jobs: + + test: + name: Test Suite + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + - uses: actions-rs/cargo@v1 + with: + command: test + args: --workspace --all-targets --features opentelemetry_0_15 + + rustfmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + components: rustfmt + - name: Check formatting + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + components: clippy + - name: Clippy check + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --all-targets --features opentelemetry_0_15 --workspace -- -D warnings + + docs: + name: Docs + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + - name: Check documentation + env: + RUSTDOCFLAGS: -D warnings + uses: actions-rs/cargo@v1 + with: + command: doc + args: --no-deps --document-private-items --features opentelemetry_0_15 --workspace + + publish-check: + name: Publish dry run + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + - uses: actions-rs/cargo@v1 + with: + command: publish + args: --dry-run --manifest-path reqwest-middleware/Cargo.toml + - uses: actions-rs/cargo@v1 + with: + command: publish + args: --dry-run --manifest-path reqwest-retry/Cargo.toml + - uses: actions-rs/cargo@v1 + with: + command: publish + args: --dry-run --manifest-path reqwest-tracing/Cargo.toml + + coverage: + name: Code coverage + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + - name: Run cargo-tarpaulin + uses: actions-rs/tarpaulin@v0.1 + with: + args: '--ignore-tests --out Lcov' + - name: Upload to Coveralls + # upload only if push + if: ${{ github.event_name == 'push' }} + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: './lcov.info' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..96ef6c0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d94e3d6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..bc1cde2 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Code of Conduct + +This project adheres to the Rust Code of Conduct, which [can be found online](https://www.rust-lang.org/conduct.html). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..19e191e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,68 @@ +# Contribution guidelines + +First off, thank you for considering contributing to reqwest-middleware. + +If your contribution is not straightforward, please first discuss the change you +wish to make by creating a new issue before making the change. + +## Reporting issues + +Before reporting an issue on the +[issue tracker](https://github.com/TrueLayer/reqwest-middleware/issues), +please check that it has not already been reported by searching for some related +keywords. + +## Pull requests + +Try to do one pull request per change. + +### Updating the changelog + +Update the changes you have made in +[CHANGELOG](https://github.com/TrueLayer/reqwest-middleware/blob/master/CHANGELOG.md) +file under the **Unreleased** section. + +Add the changes of your pull request to one of the following subsections, +depending on the types of changes defined by +[Keep a changelog](https://keepachangelog.com/en/1.0.0/): + +- `Added` for new features. +- `Changed` for changes in existing functionality. +- `Deprecated` for soon-to-be removed features. +- `Removed` for now removed features. +- `Fixed` for any bug fixes. +- `Security` in case of vulnerabilities. + +If the required subsection does not exist yet under **Unreleased**, create it! + +## Developing + +### Set up + +This is no different than other Rust projects. + +```shell +git clone https://github.com/TrueLayer/reqwest-middleware +cd reqwest-middleware +cargo test +``` + +### Useful Commands + +- Run Clippy: + + ```shell + cargo clippy + ``` + +- Check to see if there are code formatting issues + + ```shell + cargo fmt --all -- --check + ``` + +- Format the code in the project + + ```shell + cargo fmt --all + ``` diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..f84f159 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] +members = [ + "reqwest-middleware", + "reqwest-tracing", + "reqwest-retry", +] diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..1b5ec8b --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..0c9abf9 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 TrueLayer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6f5b37d --- /dev/null +++ b/README.md @@ -0,0 +1,65 @@ +# reqwest-middleware + +A crate implementing a wrapper around [reqwest](https://crates.io/crates/reqwest) +to allow for client middleware chains. + +[![Crates.io](https://img.shields.io/crates/v/reqwest-middleware.svg)](https://crates.io/crates/reqwest-middleware) +[![Docs.rs](https://docs.rs/reqwest-middleware/badge.svg)](https://docs.rs/reqwest-middleware) +[![CI](https://github.com/TrueLayer/reqwest-middleware/workflows/CI/badge.svg)](https://github.com/TrueLayer/reqwest-middleware/actions) +[![Coverage Status](https://coveralls.io/repos/github/TrueLayer/reqwest-middleware/badge.svg?branch=main&t=YKhONc)](https://coveralls.io/github/TrueLayer/reqwest-middleware?branch=main) + +## Overview + +The `reqwest-middleware` client exposes the same interface as a plain `reqwest` client, but +`ClientBuilder` exposes functionality to attach middleware: + +```rust +use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; +use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff}; +use reqwest_tracing::TracingMiddleware; + +#[tokio::main] +async fn main() { +    let retry_policy = ExponentialBackoff::builder().build_with_max_retries(3); +    let client = ClientBuilder::new(reqwest::Client::new()) +        .with(TracingMiddleware) +        .with(RetryTransientMiddleware::new_with_policy(retry_policy)) +        .build(); +    run(client).await; +} + +async fn run(client: ClientWithMiddleware) { +    // free retries! +    client +        .get("https://some-external-service.com") +        .header("foo", "bar") +        .send() +        .await +        .unwrap(); +} +``` + +## How to install + +Add `reqwest-middleware` to your dependencies + +```toml +[dependencies] +# ... +reqwest-middleware = "0.1.0" +``` + +#### License + + +Licensed under either of Apache License, Version +2.0 or MIT license at your option. + + +
+ + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. + diff --git a/reqwest-middleware/Cargo.toml b/reqwest-middleware/Cargo.toml new file mode 100644 index 0000000..e81808d --- /dev/null +++ b/reqwest-middleware/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "reqwest-middleware" +version = "0.1.0" +authors = ["Rodrigo Gryzinski "] +edition = "2018" +description = "Wrapper around reqwest to allow for client middleware chains." +repository = "https://github.com/TrueLayer/reqwest-middleware" +license = "MIT OR Apache-2.0" +keywords = ["reqwest", "http", "middleware"] +categories = ["web-programming::http-client"] +readme = "../README.md" + +[dependencies] +anyhow = "1" +async-trait = "0.1.51" +futures = "0.3" +http = "0.2" +reqwest = { version = "0.11", features = ["json", "multipart"] } +serde = "1" +thiserror = "1" +truelayer-extensions = "0.1" + +[dev-dependencies] +wiremock = "0.5" +tokio = { version = "1", features = ["macros"] } diff --git a/reqwest-middleware/src/client.rs b/reqwest-middleware/src/client.rs new file mode 100644 index 0000000..e01e8ca --- /dev/null +++ b/reqwest-middleware/src/client.rs @@ -0,0 +1,248 @@ +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use reqwest::multipart::Form; +use reqwest::{Body, Client, IntoUrl, Method, Request, Response}; +use serde::Serialize; +use std::convert::TryFrom; +use std::fmt::Display; +use std::sync::Arc; +use std::time::Duration; +use truelayer_extensions::Extensions; + +use crate::error::Result; +use crate::middleware::{Middleware, Next}; + +/// A `ClientBuilder` is used to build a [`ClientWithMiddleware`]. +/// +/// [`ClientWithMiddleware`]: struct.ClientWithMiddleware.html +pub struct ClientBuilder { + client: Client, + middleware_stack: Vec>, +} + +impl ClientBuilder { + pub fn new(client: Client) -> Self { + ClientBuilder { + client, + middleware_stack: Vec::new(), + } + } + + /// Convenience method to attach middleware. + /// + /// If you need to keep a reference to the middleware after attaching, use [`with_arc`]. + /// + /// [`with_arc`]: #method.with_arc + pub fn with(self, middleware: M) -> Self + where + M: Middleware, + { + self.with_arc(Arc::new(middleware)) + } + + /// Add middleware to the chain. [`with`] is more ergonomic if you don't need the `Arc`. + /// + /// [`with`]: #method.with + pub fn with_arc(mut self, middleware: Arc) -> Self { + self.middleware_stack.push(middleware); + self + } + + /// Returns a `ClientWithMiddleware` using this builder configuration. + pub fn build(self) -> ClientWithMiddleware { + ClientWithMiddleware::new(self.client, self.middleware_stack) + } +} + +/// `ClientWithMiddleware` is a wrapper around [`reqwest::Client`] which runs middleware on every +/// request. +/// +/// [`reqwest::Client`]: https://docs.rs/reqwest/0.10.8/reqwest/struct.Client.html +#[derive(Clone)] +pub struct ClientWithMiddleware { + inner: reqwest::Client, + middleware_stack: Box<[Arc]>, +} + +impl ClientWithMiddleware { + /// See [`ClientBuilder`] for a more ergonomic way to build `ClientWithMiddleware` instances. + /// + /// [`ClientBuilder`]: struct.ClientBuilder.html + pub fn new(client: Client, middleware_stack: T) -> Self + where + T: Into]>>, + { + ClientWithMiddleware { + inner: client, + middleware_stack: middleware_stack.into(), + } + } + + /// See [`Client::get`](https://docs.rs/reqwest/latest/reqwest/struct.Client.html#method.get) + pub fn get(&self, url: U) -> RequestBuilder { + self.request(Method::GET, url) + } + + /// See [`Client::post`](https://docs.rs/reqwest/latest/reqwest/struct.Client.html#method.post) + pub fn post(&self, url: U) -> RequestBuilder { + self.request(Method::POST, url) + } + + /// See [`Client::put`](https://docs.rs/reqwest/latest/reqwest/struct.Client.html#method.put) + pub fn put(&self, url: U) -> RequestBuilder { + self.request(Method::PUT, url) + } + + /// See + /// [`Client::patch`](https://docs.rs/reqwest/latest/reqwest/struct.Client.html#method.patch) + pub fn patch(&self, url: U) -> RequestBuilder { + self.request(Method::PATCH, url) + } + + /// See + /// [`Client::delete`](https://docs.rs/reqwest/latest/reqwest/struct.Client.html#method.delete) + pub fn delete(&self, url: U) -> RequestBuilder { + self.request(Method::DELETE, url) + } + + /// See [`Client::head`](https://docs.rs/reqwest/latest/reqwest/struct.Client.html#method.head) + pub fn head(&self, url: U) -> RequestBuilder { + self.request(Method::HEAD, url) + } + + /// See + /// [`Client::request`](https://docs.rs/reqwest/latest/reqwest/struct.Client.html#method.request) + pub fn request(&self, method: Method, url: U) -> RequestBuilder { + RequestBuilder { + inner: self.inner.request(method, url), + client: self.clone(), + } + } + + /// See + /// [`Client::execute`](https://docs.rs/reqwest/latest/reqwest/struct.Client.html#method.execute) + pub async fn execute(&self, req: Request) -> Result { + let next = Next::new(&self.inner, &self.middleware_stack); + let mut ext = Extensions::new(); + next.run(req, &mut ext).await + } +} + +/// Create a `ClientWithMiddleware` without any middleware. +impl From for ClientWithMiddleware { + fn from(client: Client) -> Self { + ClientWithMiddleware { + inner: client, + middleware_stack: Box::new([]), + } + } +} + +/// This is a wrapper around [`reqwest::RequestBuilder`] exposing the same API. +/// +/// [`reqwest::RequestBuilder`]: https://docs.rs/reqwest/0.10.8/reqwest/struct.RequestBuilder.html +#[must_use = "RequestBuilder does nothing until you 'send' it"] +pub struct RequestBuilder { + inner: reqwest::RequestBuilder, + client: ClientWithMiddleware, +} + +impl RequestBuilder { + pub fn header(self, key: K, value: V) -> Self + where + HeaderName: TryFrom, + >::Error: Into, + HeaderValue: TryFrom, + >::Error: Into, + { + RequestBuilder { + inner: self.inner.header(key, value), + client: self.client, + } + } + + pub fn headers(self, headers: HeaderMap) -> Self { + RequestBuilder { + inner: self.inner.headers(headers), + client: self.client, + } + } + + pub fn basic_auth(self, username: U, password: Option

) -> Self + where + U: Display, + P: Display, + { + RequestBuilder { + inner: self.inner.basic_auth(username, password), + client: self.client, + } + } + + pub fn bearer_auth(self, token: T) -> Self + where + T: Display, + { + RequestBuilder { + inner: self.inner.bearer_auth(token), + client: self.client, + } + } + + pub fn body>(self, body: T) -> Self { + RequestBuilder { + inner: self.inner.body(body), + client: self.client, + } + } + + pub fn timeout(self, timeout: Duration) -> Self { + RequestBuilder { + inner: self.inner.timeout(timeout), + client: self.client, + } + } + + pub fn multipart(self, multipart: Form) -> Self { + RequestBuilder { + inner: self.inner.multipart(multipart), + client: self.client, + } + } + + pub fn query(self, query: &T) -> Self { + RequestBuilder { + inner: self.inner.query(query), + client: self.client, + } + } + + pub fn form(self, form: &T) -> Self { + RequestBuilder { + inner: self.inner.form(form), + client: self.client, + } + } + + pub fn json(self, json: &T) -> Self { + RequestBuilder { + inner: self.inner.json(json), + client: self.client, + } + } + + pub fn build(self) -> reqwest::Result { + self.inner.build() + } + + pub async fn send(self) -> Result { + let req = self.inner.build()?; + self.client.execute(req).await + } + + pub fn try_clone(self) -> Option { + let client = self.client; + self.inner + .try_clone() + .map(|inner| RequestBuilder { inner, client }) + } +} diff --git a/reqwest-middleware/src/error.rs b/reqwest-middleware/src/error.rs new file mode 100644 index 0000000..c276578 --- /dev/null +++ b/reqwest-middleware/src/error.rs @@ -0,0 +1,22 @@ +use thiserror::Error; + +pub type Result = std::result::Result; + +#[derive(Error, Debug)] +pub enum Error { + /// There was an error running some middleware + #[error("Middleware error: {0}")] + Middleware(#[from] anyhow::Error), + /// Error from the underlying reqwest client + #[error("Request error: {0}")] + Reqwest(#[from] reqwest::Error), +} + +impl Error { + pub fn middleware(err: E) -> Self + where + E: 'static + Send + Sync + std::error::Error, + { + Error::Middleware(err.into()) + } +} diff --git a/reqwest-middleware/src/lib.rs b/reqwest-middleware/src/lib.rs new file mode 100644 index 0000000..28531cc --- /dev/null +++ b/reqwest-middleware/src/lib.rs @@ -0,0 +1,51 @@ +//! This crate provides [`ClientWithMiddleware`], a wrapper around [`reqwest::Client`] with the +//! ability to attach middleware which runs on every request. +//! +//! You'll want to instantiate [`ClientWithMiddleware`] using [`ClientBuilder`], then you can +//! attach your middleware using [`with`], finalize it with [`build`] and from then on sending +//! requests is the same as with reqwest: +//! +//! ``` +//! use reqwest::{Client, Request, Response}; +//! use reqwest_middleware::{ClientBuilder, Middleware, Next, Result}; +//! use truelayer_extensions::Extensions; +//! +//! struct LoggingMiddleware; +//! +//! #[async_trait::async_trait] +//! impl Middleware for LoggingMiddleware { +//! async fn handle( +//! &self, +//! req: Request, +//! extensions: &mut Extensions, +//! next: Next<'_>, +//! ) -> Result { +//! println!("Request started {:?}", req); +//! let res = next.run(req, extensions).await; +//! println!("Result: {:?}", res); +//! res +//! } +//! } +//! +//! async fn run() { +//! let reqwest_client = Client::builder().build().unwrap(); +//! let client = ClientBuilder::new(reqwest_client) +//! .with(LoggingMiddleware) +//! .build(); +//! let resp = client.get("https://truelayer.com").send().await.unwrap(); +//! println!("TrueLayer page HTML: {}", resp.text().await.unwrap()); +//! } +//! ``` +//! +//! [`build`]: struct.ClientBuilder.html#method.build +//! [`ClientBuilder`]: struct.ClientBuilder.html +//! [`ClientWithMiddleware`]: struct.ClientWithMiddleware.html +//! [`reqwest::Client`]: https://docs.rs/reqwest/0.10.8/reqwest/struct.Client.html +//! [`with`]: struct.ClientBuilder.html#method.with +mod client; +mod error; +mod middleware; + +pub use client::{ClientBuilder, ClientWithMiddleware, RequestBuilder}; +pub use error::{Error, Result}; +pub use middleware::{Middleware, Next}; diff --git a/reqwest-middleware/src/middleware.rs b/reqwest-middleware/src/middleware.rs new file mode 100644 index 0000000..52a8de0 --- /dev/null +++ b/reqwest-middleware/src/middleware.rs @@ -0,0 +1,100 @@ +use futures::future::{BoxFuture, FutureExt, TryFutureExt}; +use reqwest::{Client, Request, Response}; +use std::sync::Arc; +use truelayer_extensions::Extensions; + +use crate::error::{Error, Result}; + +/// When attached to a [`ClientWithMiddleware`] (generally using [`with`]), middleware is run +/// whenever the client issues a request, in the order it was attached. +/// +/// # Example +/// +/// ``` +/// use reqwest::{Client, Request, Response}; +/// use reqwest_middleware::{ClientBuilder, Middleware, Next, Result}; +/// use truelayer_extensions::Extensions; +/// +/// struct TransparentMiddleware; +/// +/// #[async_trait::async_trait] +/// impl Middleware for TransparentMiddleware { +/// async fn handle( +/// &self, +/// req: Request, +/// extensions: &mut Extensions, +/// next: Next<'_>, +/// ) -> Result { +/// next.run(req, extensions).await +/// } +/// } +/// ``` +/// +/// [`ClientWithMiddleware`]: struct.ClientWithMiddleware.html +/// [`Extensions`]: TODO +/// [`with`]: struct.ClientBuilder.html#method.with +#[async_trait::async_trait] +pub trait Middleware: 'static + Send + Sync { + /// Invoked with a request before sending it. If you want to continue processing the request, + /// you should explicitly call `next.run(req, extensions)`. + /// + /// If you need to forward data down the middleware stack, you can use the `extensions` + /// argument. + async fn handle( + &self, + req: Request, + extensions: &mut Extensions, + next: Next<'_>, + ) -> Result; +} + +#[async_trait::async_trait] +impl Middleware for F +where + F: Send + + Sync + + 'static + + for<'a> Fn(Request, &'a mut Extensions, Next<'a>) -> BoxFuture<'a, Result>, +{ + async fn handle( + &self, + req: Request, + extensions: &mut Extensions, + next: Next<'_>, + ) -> Result { + (self)(req, extensions, next).await + } +} + +/// Next encapsulates the remaining middleware chain to run in [`Middleware::handle`]. You can +/// forward the request down the chain with [`run`]. +/// +/// [`Middleware::handle`]: trait.Middleware.html#tymethod.handle +/// [`run`]: #method.run +#[derive(Clone)] +pub struct Next<'a> { + client: &'a Client, + middlewares: &'a [Arc], +} + +impl<'a> Next<'a> { + pub(crate) fn new(client: &'a Client, middlewares: &'a [Arc]) -> Self { + Next { + client, + middlewares, + } + } + + pub fn run( + mut self, + req: Request, + extensions: &'a mut Extensions, + ) -> BoxFuture<'a, Result> { + if let Some((current, rest)) = self.middlewares.split_first() { + self.middlewares = rest; + current.handle(req, extensions, self).boxed() + } else { + self.client.execute(req).map_err(Error::from).boxed() + } + } +} diff --git a/reqwest-retry/.gitignore b/reqwest-retry/.gitignore new file mode 100644 index 0000000..96ef6c0 --- /dev/null +++ b/reqwest-retry/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/reqwest-retry/CHANGELOG.md b/reqwest-retry/CHANGELOG.md new file mode 100644 index 0000000..d94e3d6 --- /dev/null +++ b/reqwest-retry/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] diff --git a/reqwest-retry/Cargo.toml b/reqwest-retry/Cargo.toml new file mode 100644 index 0000000..b8bb879 --- /dev/null +++ b/reqwest-retry/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "reqwest-retry" +version = "0.1.0" +authors = ["Rodrigo Gryzinski "] +edition = "2018" +description = "Retry middleware for reqwest." +repository = "https://github.com/TrueLayer/reqwest-middleware" +license = "MIT OR Apache-2.0" +keywords = ["reqwest", "http", "middleware", "retry"] +categories = ["web-programming::http-client"] + +[dependencies] +reqwest-middleware = { version = "0.1", path = "../reqwest-middleware" } + +anyhow = "1" +async-trait = "0.1.51" +chrono = "0.4" +futures = "0.3" +http = "0.2" +retry-policies = "0.1" +reqwest = "0.11" +tokio = { version = "1.6", features = ["time"] } +tracing = "0.1.26" +truelayer-extensions = "0.1" + +[dev-dependencies] +wiremock = "0.5" +tokio = { version = "1", features = ["macros"] } +paste = "1" diff --git a/reqwest-retry/README.md b/reqwest-retry/README.md new file mode 100644 index 0000000..9376295 --- /dev/null +++ b/reqwest-retry/README.md @@ -0,0 +1,41 @@ +# reqwest-retry + +Retry middleware implementation for +[`reqwest-middleware`](https://crates.io/crates/reqwest-middleware). + +[![Crates.io](https://img.shields.io/crates/v/reqwest-retry.svg)](https://crates.io/crates/reqwest-retry) +[![Docs.rs](https://docs.rs/reqwest-retry/badge.svg)](https://docs.rs/reqwest-retry) +[![CI](https://github.com/TrueLayer/reqwest-middleware/workflows/CI/badge.svg)](https://github.com/TrueLayer/reqwest-middleware/actions) +[![Coverage Status](https://coveralls.io/repos/github/TrueLayer/reqwest-middleware/badge.svg?branch=main&t=UWgSpm)](https://coveralls.io/github/TrueLayer/reqwest-middleware?branch=main) + +## Overview + +Build `RetryTransientMiddleware` from a `RetryPolicy`, then attach it to a +`reqwest_middleware::ClientBuilder`. +[`retry-policies::policies`](https://crates.io/crates/retry-policies) is reexported under +`reqwest_retry::policies` for convenience. + +## How to install + +Add `reqwest-retry` to your dependencies + +```toml +[dependencies] +# ... +reqwest-retry = "0.1.0" +``` + +#### License + + +Licensed under either of Apache License, Version +2.0 or MIT license at your option. + + +
+ + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. + diff --git a/reqwest-retry/src/lib.rs b/reqwest-retry/src/lib.rs new file mode 100644 index 0000000..06130ab --- /dev/null +++ b/reqwest-retry/src/lib.rs @@ -0,0 +1,7 @@ +mod middleware; +mod retryable; + +pub use retry_policies::policies; + +pub use middleware::RetryTransientMiddleware; +pub use retryable::Retryable; diff --git a/reqwest-retry/src/middleware.rs b/reqwest-retry/src/middleware.rs new file mode 100644 index 0000000..cbbe930 --- /dev/null +++ b/reqwest-retry/src/middleware.rs @@ -0,0 +1,139 @@ +//! `RetryTransientMiddleware` implements retrying requests on transient errors. + +use crate::retryable::Retryable; +use anyhow::anyhow; +use chrono::Utc; +use reqwest::{Request, Response}; +use reqwest_middleware::{Error, Middleware, Next, Result}; +use retry_policies::RetryPolicy; +use truelayer_extensions::Extensions; + +/// We limit the number of retries to a maximum of `10` to avoid stack-overflow issues due to the recursion. +static MAXIMUM_NUMBER_OF_RETRIES: u32 = 10; + +/// `RetryTransientMiddleware` offers retry logic for requests that fail in a transient manner +/// and can be safely executed again. +/// +/// Currently, it allows setting a [RetryPolicy][retry_policies::RetryPolicy] algorithm for calculating the __wait_time__ +/// between each request retry. +/// +///```rust +/// use reqwest_middleware::ClientBuilder; +/// use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff}; +/// use reqwest::Client; +/// +/// // We create a ExponentialBackoff retry policy which implements `RetryPolicy`. +/// let retry_policy = ExponentialBackoff { +/// /// How many times the policy will tell the middleware to retry the request. +/// max_n_retries: 3, +/// max_retry_interval: std::time::Duration::from_millis(30), +/// min_retry_interval: std::time::Duration::from_millis(100), +/// backoff_exponent: 2, +/// }; +/// +/// let retry_transient_middleware = RetryTransientMiddleware::new_with_policy(retry_policy); +/// let client = ClientBuilder::new(Client::new()).with(retry_transient_middleware).build(); +///``` +/// +pub struct RetryTransientMiddleware { + retry_policy: T, +} + +impl RetryTransientMiddleware { + /// Construct `RetryTransientMiddleware` with a [retry_policy][retry_policies::RetryPolicy]. + pub fn new_with_policy(retry_policy: T) -> Self { + Self { retry_policy } + } +} + +#[async_trait::async_trait] +impl Middleware for RetryTransientMiddleware { + async fn handle( + &self, + req: Request, + extensions: &mut Extensions, + next: Next<'_>, + ) -> Result { + // TODO: Ideally we should create a new instance of the `Extensions` map to pass + // downstream. This will guard against previous retries poluting `Extensions`. + // That is, we only return what's populated in the typemap for the last retry attempt + // and copy those into the the `global` Extensions map. + self.execute_with_retry_recursive(req, next, extensions, 0) + .await + } +} + +impl RetryTransientMiddleware { + /// **RECURSIVE**. + /// + /// SAFETY: The condition for termination is the number of retries + /// set on the `RetryOption` object which is capped to 10 therefore + /// we can know that this will not cause a overflow of the stack. + /// + /// This function will try to execute the request, if it fails + /// with an error classified as transient it will call itself + /// to retry the request. + /// + /// NOTE: This function is not async because calling an async function + /// recursively is not allowed. + /// + fn execute_with_retry_recursive<'a>( + &'a self, + req: Request, + next: Next<'a>, + mut ext: &'a mut Extensions, + n_past_retries: u32, + ) -> futures::future::BoxFuture<'a, Result> { + Box::pin(async move { + // Cloning the request object before-the-fact is not ideal.. + // However, if the body of the request is not static, e.g of type `Bytes`, + // the Clone operation should be of constant complexity and not O(N) + // since the byte abstraction is a shared pointer over a buffer. + let duplicate_request = req.try_clone().ok_or_else(|| { + Error::Middleware(anyhow!( + "Request object is not clonable. Are you passing a streaming body?".to_string() + )) + })?; + + let cloned_next = next.clone(); + + let result = next.run(req, &mut ext).await; + + // We classify the response which will return None if not + // errors were returned. + match Retryable::from_reqwest_response(&result) { + Some(retryable) + if retryable == Retryable::Transient + && n_past_retries < MAXIMUM_NUMBER_OF_RETRIES => + { + // If the response failed and the error type was transient + // we can safely try to retry the request. + let retry_decicion = self.retry_policy.should_retry(n_past_retries); + if let retry_policies::RetryDecision::Retry { execute_after } = retry_decicion { + let duration = (execute_after - Utc::now()) + .to_std() + .map_err(Error::middleware)?; + // Sleep the requested amount before we try again. + tracing::warn!( + "Retry attempt #{}. Sleeping {:?} before the next attempt", + n_past_retries, + duration + ); + tokio::time::sleep(duration).await; + + self.execute_with_retry_recursive( + duplicate_request, + cloned_next, + ext, + n_past_retries + 1, + ) + .await + } else { + result + } + } + Some(_) | None => result, + } + }) + } +} diff --git a/reqwest-retry/src/retryable.rs b/reqwest-retry/src/retryable.rs new file mode 100644 index 0000000..0087f3e --- /dev/null +++ b/reqwest-retry/src/retryable.rs @@ -0,0 +1,68 @@ +use http::StatusCode; +use reqwest_middleware::Error; + +/// Classification of an error/status returned by request. +#[derive(PartialEq, Eq)] +pub enum Retryable { + /// The failure was due to something tha might resolve in the future. + Transient, + /// Unresolvable error. + Fatal, +} + +impl Retryable { + /// Try to map a `reqwest` response into `Retryable`. + /// + /// Returns `None` if the response object does not contain any errors. + /// + pub fn from_reqwest_response(res: &Result) -> Option { + match res { + Ok(success) => { + let status = success.status(); + if status.is_server_error() { + Some(Retryable::Transient) + } else if status.is_client_error() + && status != StatusCode::REQUEST_TIMEOUT + && status != StatusCode::TOO_MANY_REQUESTS + { + Some(Retryable::Fatal) + } else if status.is_success() { + None + } else if status == StatusCode::REQUEST_TIMEOUT + || status == StatusCode::TOO_MANY_REQUESTS + { + Some(Retryable::Transient) + } else { + Some(Retryable::Fatal) + } + } + Err(error) => match error { + // If something fails in the middleware we're screwed. + Error::Middleware(_) => Some(Retryable::Fatal), + Error::Reqwest(error) => { + if error.is_timeout() || error.is_connect() { + Some(Retryable::Transient) + } else if error.is_body() + || error.is_decode() + || error.is_request() + || error.is_builder() + || error.is_redirect() + { + Some(Retryable::Fatal) + } else { + // We omit checking if error.is_status() since we check that already. + // However, if Response::error_for_status is used the status will still + // remain in the response object. + None + } + } + }, + } + } +} + +impl From<&reqwest::Error> for Retryable { + fn from(_status: &reqwest::Error) -> Retryable { + Retryable::Transient + } +} diff --git a/reqwest-retry/tests/retry.rs b/reqwest-retry/tests/retry.rs new file mode 100644 index 0000000..ff721f4 --- /dev/null +++ b/reqwest-retry/tests/retry.rs @@ -0,0 +1,215 @@ +use paste::paste; +use reqwest::Client; +use reqwest::StatusCode; +use reqwest_middleware::ClientBuilder; +use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; +use std::sync::{ + atomic::{AtomicU32, Ordering}, + Arc, +}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, Respond, ResponseTemplate}; + +pub struct RetryResponder(Arc, u32, u16); + +impl RetryResponder { + fn new(retries: u32, status_code: u16) -> Self { + Self(Arc::new(AtomicU32::new(0)), retries, status_code) + } +} + +impl Respond for RetryResponder { + fn respond(&self, _request: &wiremock::Request) -> ResponseTemplate { + let mut retries = self.0.load(Ordering::SeqCst); + retries += 1; + self.0.store(retries, Ordering::SeqCst); + + if retries + 1 >= self.1 { + ResponseTemplate::new(200) + } else { + ResponseTemplate::new(self.2) + } + } +} + +macro_rules! assert_retry_succeeds_inner { + ($x:tt, $name:ident, $status:expr, $retry:tt, $exact:tt, $responder:expr) => { + #[tokio::test] + async fn $name() { + let server = MockServer::start().await; + let retry_amount: u32 = $retry; + Mock::given(method("GET")) + .and(path("/foo")) + .respond_with($responder) + .expect($exact) + .mount(&server) + .await; + + let reqwest_client = Client::builder().build().unwrap(); + let client = ClientBuilder::new(reqwest_client) + .with(RetryTransientMiddleware::new_with_policy( + ExponentialBackoff { + max_n_retries: retry_amount, + max_retry_interval: std::time::Duration::from_millis(30), + min_retry_interval: std::time::Duration::from_millis(100), + backoff_exponent: 2, + }, + )) + .build(); + + let resp = client + .get(&format!("{}/foo", server.uri())) + .send() + .await + .expect("call failed"); + + assert_eq!(resp.status(), $status); + } + }; +} + +macro_rules! assert_retry_succeeds { + ($x:tt, $status:expr) => { + paste! { + assert_retry_succeeds_inner!($x, [], $status, 3, 2, RetryResponder::new(3 as u32, $x)); + } + }; +} + +macro_rules! assert_no_retry { + ($x:tt, $status:expr) => { + paste! { + assert_retry_succeeds_inner!($x, [], $status, 1, 1, ResponseTemplate::new($x)); + } + }; +} + +// 2xx. +assert_no_retry!(200, StatusCode::OK); +assert_no_retry!(201, StatusCode::CREATED); +assert_no_retry!(202, StatusCode::ACCEPTED); +assert_no_retry!(203, StatusCode::NON_AUTHORITATIVE_INFORMATION); +assert_no_retry!(204, StatusCode::NO_CONTENT); +assert_no_retry!(205, StatusCode::RESET_CONTENT); +assert_no_retry!(206, StatusCode::PARTIAL_CONTENT); +assert_no_retry!(207, StatusCode::MULTI_STATUS); +assert_no_retry!(226, StatusCode::IM_USED); + +// 3xx. +assert_no_retry!(300, StatusCode::MULTIPLE_CHOICES); +assert_no_retry!(301, StatusCode::MOVED_PERMANENTLY); +assert_no_retry!(302, StatusCode::FOUND); +assert_no_retry!(303, StatusCode::SEE_OTHER); +assert_no_retry!(304, StatusCode::NOT_MODIFIED); +assert_no_retry!(307, StatusCode::TEMPORARY_REDIRECT); +assert_no_retry!(308, StatusCode::PERMANENT_REDIRECT); + +// 5xx. +assert_retry_succeeds!(500, StatusCode::OK); +assert_retry_succeeds!(501, StatusCode::OK); +assert_retry_succeeds!(502, StatusCode::OK); +assert_retry_succeeds!(503, StatusCode::OK); +assert_retry_succeeds!(504, StatusCode::OK); +assert_retry_succeeds!(505, StatusCode::OK); +assert_retry_succeeds!(506, StatusCode::OK); +assert_retry_succeeds!(507, StatusCode::OK); +assert_retry_succeeds!(508, StatusCode::OK); +assert_retry_succeeds!(510, StatusCode::OK); +assert_retry_succeeds!(511, StatusCode::OK); +// 4xx. +assert_no_retry!(400, StatusCode::BAD_REQUEST); +assert_no_retry!(401, StatusCode::UNAUTHORIZED); +assert_no_retry!(402, StatusCode::PAYMENT_REQUIRED); +assert_no_retry!(403, StatusCode::FORBIDDEN); +assert_no_retry!(404, StatusCode::NOT_FOUND); +assert_no_retry!(405, StatusCode::METHOD_NOT_ALLOWED); +assert_no_retry!(406, StatusCode::NOT_ACCEPTABLE); +assert_no_retry!(407, StatusCode::PROXY_AUTHENTICATION_REQUIRED); +assert_retry_succeeds!(408, StatusCode::OK); +assert_no_retry!(409, StatusCode::CONFLICT); +assert_no_retry!(410, StatusCode::GONE); +assert_no_retry!(411, StatusCode::LENGTH_REQUIRED); +assert_no_retry!(412, StatusCode::PRECONDITION_FAILED); +assert_no_retry!(413, StatusCode::PAYLOAD_TOO_LARGE); +assert_no_retry!(414, StatusCode::URI_TOO_LONG); +assert_no_retry!(415, StatusCode::UNSUPPORTED_MEDIA_TYPE); +assert_no_retry!(416, StatusCode::RANGE_NOT_SATISFIABLE); +assert_no_retry!(417, StatusCode::EXPECTATION_FAILED); +assert_no_retry!(418, StatusCode::IM_A_TEAPOT); +assert_no_retry!(421, StatusCode::MISDIRECTED_REQUEST); +assert_no_retry!(422, StatusCode::UNPROCESSABLE_ENTITY); +assert_no_retry!(423, StatusCode::LOCKED); +assert_no_retry!(424, StatusCode::FAILED_DEPENDENCY); +assert_no_retry!(426, StatusCode::UPGRADE_REQUIRED); +assert_no_retry!(428, StatusCode::PRECONDITION_REQUIRED); +assert_retry_succeeds!(429, StatusCode::OK); +assert_no_retry!(431, StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE); +assert_no_retry!(451, StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS); + +// We assert that we cap retries at 10, which means that we will +// get 11 calls to the RetryResponder. +assert_retry_succeeds_inner!( + 500, + assert_maximum_retries_is_not_exceeded, + StatusCode::INTERNAL_SERVER_ERROR, + 100, + 11, + RetryResponder::new(100_u32, 500) +); + +pub struct RetryTimeoutResponder(Arc, u32, std::time::Duration); + +impl RetryTimeoutResponder { + fn new(retries: u32, initial_timeout: std::time::Duration) -> Self { + Self(Arc::new(AtomicU32::new(0)), retries, initial_timeout) + } +} + +impl Respond for RetryTimeoutResponder { + fn respond(&self, _request: &wiremock::Request) -> ResponseTemplate { + let mut retries = self.0.load(Ordering::SeqCst); + retries += 1; + self.0.store(retries, Ordering::SeqCst); + + if retries + 1 >= self.1 { + ResponseTemplate::new(200) + } else { + ResponseTemplate::new(500).set_delay(self.2) + } + } +} + +#[tokio::test] +async fn assert_retry_on_request_timeout() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/foo")) + .respond_with(RetryTimeoutResponder::new( + 3, + std::time::Duration::from_millis(1000), + )) + .expect(2) + .mount(&server) + .await; + + let reqwest_client = Client::builder().build().unwrap(); + let client = ClientBuilder::new(reqwest_client) + .with(RetryTransientMiddleware::new_with_policy( + ExponentialBackoff { + max_n_retries: 3, + max_retry_interval: std::time::Duration::from_millis(30), + min_retry_interval: std::time::Duration::from_millis(100), + backoff_exponent: 2, + }, + )) + .build(); + + let resp = client + .get(&format!("{}/foo", server.uri())) + .timeout(std::time::Duration::from_millis(10)) + .send() + .await + .expect("call failed"); + + assert_eq!(resp.status(), 200); +} diff --git a/reqwest-tracing/CHANGELOG.md b/reqwest-tracing/CHANGELOG.md new file mode 100644 index 0000000..d94e3d6 --- /dev/null +++ b/reqwest-tracing/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] diff --git a/reqwest-tracing/Cargo.toml b/reqwest-tracing/Cargo.toml new file mode 100644 index 0000000..eb1017c --- /dev/null +++ b/reqwest-tracing/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "reqwest-tracing" +version = "0.1.0" +authors = ["Rodrigo Gryzinski "] +edition = "2018" +description = "Opentracing middleware for reqwest." +repository = "https://github.com/TrueLayer/reqwest-middleware" +license = "MIT OR Apache-2.0" +keywords = ["reqwest", "http", "middleware", "opentracing", "tracing"] +categories = ["web-programming::http-client"] + +[features] +opentelemetry_0_13 = ["opentelemetry_0_13_pkg", "tracing-opentelemetry_0_12_pkg"] +opentelemetry_0_14 = ["opentelemetry_0_14_pkg", "tracing-opentelemetry_0_13_pkg"] +opentelemetry_0_15 = ["opentelemetry_0_15_pkg", "tracing-opentelemetry_0_14_pkg"] + +[dependencies] +reqwest-middleware = { version = "0.1", path = "../reqwest-middleware" } + +async-trait = "0.1.51" +reqwest = "0.11" +tokio = { version = "1.6", features = ["time"] } +tracing = "0.1.26" +truelayer-extensions = "0.1" + +opentelemetry_0_13_pkg = { package = "opentelemetry", version = "0.13", optional = true } +opentelemetry_0_14_pkg = { package = "opentelemetry", version = "0.14", optional = true } +opentelemetry_0_15_pkg = { package = "opentelemetry", version = "0.15", optional = true } +tracing-opentelemetry_0_12_pkg = { package = "tracing-opentelemetry",version = "0.12", optional = true } +tracing-opentelemetry_0_13_pkg = { package = "tracing-opentelemetry", version = "0.13", optional = true } +tracing-opentelemetry_0_14_pkg = { package = "tracing-opentelemetry",version = "0.14", optional = true } + +[dev-dependencies] +wiremock = "0.5" +tokio = { version = "1", features = ["macros"] } diff --git a/reqwest-tracing/README.md b/reqwest-tracing/README.md new file mode 100644 index 0000000..e9e5468 --- /dev/null +++ b/reqwest-tracing/README.md @@ -0,0 +1,71 @@ +# reqwest-tracing + +Opentracing middleware implementation for +[`reqwest-middleware`](https://crates.io/crates/reqwest-middleware). + +[![Crates.io](https://img.shields.io/crates/v/reqwest-tracing.svg)](https://crates.io/crates/reqwest-tracing) +[![Docs.rs](https://docs.rs/reqwest-tracing/badge.svg)](https://docs.rs/reqwest-tracing) +[![CI](https://github.com/TrueLayer/reqwest-middleware/workflows/CI/badge.svg)](https://github.com/TrueLayer/reqwest-middleware/actions) +[![Coverage Status](https://coveralls.io/repos/github/TrueLayer/reqwest-middleware/badge.svg?branch=main&t=UWgSpm)](https://coveralls.io/github/TrueLayer/reqwest-middleware?branch=main) + +## Overview + +Attach `TracingMiddleware` to your client to automatically trace HTTP requests: + +```rust +use opentelemetry::exporter::trace::stdout; +use reqwest_middleware::ClientBuilder; +use reqwest_tracing::TracingMiddleware; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::Registry; + +#[tokio::main] +async fn main() { + let (tracer, _) = stdout::new_pipeline().install(); + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + let subscriber = Registry::default().with(telemetry); + tracing::subscriber::set_global_default(subscriber).unwrap(); + + run().await; +} + +async fun run() { + let client = ClientBuilder::new(reqwest::Client::new()) + .with(TracingMiddleware) + .build();` + + client.get("https://truelayer.com").send().await.unwrap(); +} +``` + +See the [`tracing`](https://crates.io/crates/tracing) crate for more information on how to set up a +tracing subscriber to make use of the spans. + +## How to install + +Add `reqwest-tracing` to your dependencies. Optionally enable opentelemetry integration by enabling +an opentelemetry version feature: + +```toml +[dependencies] +# ... +reqwest-tracing = { version = "0.1.0", features = ["opentelemetry_0_15"] } +``` + +Available opentelemetry features are `opentelemetry_0_15`, `opentelemetry_0_14` and +`opentelemetry_0_13`. + +#### License + + +Licensed under either of Apache License, Version +2.0 or MIT license at your option. + + +
+ + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. + diff --git a/reqwest-tracing/src/lib.rs b/reqwest-tracing/src/lib.rs new file mode 100644 index 0000000..fc7887e --- /dev/null +++ b/reqwest-tracing/src/lib.rs @@ -0,0 +1,9 @@ +mod middleware; +#[cfg(any( + feature = "opentelemetry_0_13", + feature = "opentelemetry_0_14", + feature = "opentelemetry_0_15" +))] +mod otel; + +pub use middleware::TracingMiddleware; diff --git a/reqwest-tracing/src/middleware.rs b/reqwest-tracing/src/middleware.rs new file mode 100644 index 0000000..9cb65a8 --- /dev/null +++ b/reqwest-tracing/src/middleware.rs @@ -0,0 +1,129 @@ +use reqwest::header::{HeaderMap, HeaderValue}; +use reqwest::{Request, Response, StatusCode as RequestStatusCode}; +use reqwest_middleware::{Error, Middleware, Next, Result}; +use truelayer_extensions::Extensions; + +/// Middleware for tracing requests using the current Opentelemetry Context. +pub struct TracingMiddleware; + +#[async_trait::async_trait] +impl Middleware for TracingMiddleware { + async fn handle( + &self, + req: Request, + extensions: &mut Extensions, + next: Next<'_>, + ) -> Result { + let request_span = { + let method = req.method(); + let scheme = req.url().scheme(); + let host = req.url().host_str().unwrap_or(""); + let host_port = req.url().port().unwrap_or(0) as i64; + let path = req.url().path(); + let otel_name = format!("{} {}", method, path); + + tracing::info_span!( + "HTTP request", + http.method = %method, + http.scheme = %scheme, + http.host = %host, + net.host.port = %host_port, + otel.kind = "client", + otel.name = %otel_name, + otel.status_code = tracing::field::Empty, + http.user_agent = tracing::field::Empty, + http.status_code = tracing::field::Empty, + error.message = tracing::field::Empty, + error.cause_chain = tracing::field::Empty, + ) + }; + + // Adds tracing headers to the given request to propagate the OpenTracing context to downstream revivers of the request. + // Spans added by downstream consumers will be part of the same trace. + #[cfg(any( + feature = "opentelemetry_0_13", + feature = "opentelemetry_0_14", + feature = "opentelemetry_0_15" + ))] + let req = crate::otel::inject_opentracing_context_into_request(&request_span, req); + + // Run the request + let outcome = next.run(req, extensions).await; + match &outcome { + Ok(response) => { + // The request ran successfully + let span_status = get_span_status(response.status()); + let status_code = response.status().as_u16() as i64; + let user_agent = get_header_value("user_agent", response.headers()); + if let Some(span_status) = span_status { + request_span.record("otel.status_code", &span_status); + } + request_span.record("http.status_code", &status_code); + request_span.record("http.user_agent", &user_agent.as_str()); + } + Err(e) => { + // The request didn't run successfully + let error_message = e.to_string(); + let error_cause_chain = format!("{:?}", e); + request_span.record("otel.status_code", &"ERROR"); + request_span.record("error.message", &error_message.as_str()); + request_span.record("error.cause_chain", &error_cause_chain.as_str()); + if let Error::Reqwest(e) = e { + request_span.record( + "http.status_code", + &e.status() + .map(|s| s.to_string()) + .unwrap_or_else(|| "".to_string()) + .as_str(), + ); + } + } + } + outcome + } +} + +fn get_header_value(key: &str, headers: &HeaderMap) -> String { + let header_default = &HeaderValue::from_static(""); + format!("{:?}", headers.get(key).unwrap_or(header_default)).replace("\"", "") +} + +/// HTTP Mapping +// | HTTP code | Span status code | +// |-------------------------|-----------------------| +// | 100...299 | `Ok` | +// | 3xx redirect codes | `DeadlineExceeded` in case of loop (see above) [1], otherwise `Ok` | +// | 401 Unauthorized ⚠ | `Unauthenticated` ⚠ (Unauthorized actually means unauthenticated according to [RFC 7235][rfc-unauthorized]) | +// | 403 Forbidden | `PermissionDenied` | +// | 404 Not Found | `NotFound` | +// | 429 Too Many Requests | `ResourceExhausted` | +// | Other 4xx code | `InvalidArgument` [1] | +// | 501 Not Implemented | `Unimplemented` | +// | 503 Service Unavailable | `Unavailable` | +// | 504 Gateway Timeout | `DeadlineExceeded` | +// | Other 5xx code | `InternalError` [1] | +// | Any status code the client fails to interpret (e.g., 093 or 573) | `UnknownError` | +/// +/// Maps the the http status to an Opentelemetry span status following the the specified convention above. +fn get_span_status(request_status: RequestStatusCode) -> Option<&'static str> { + match request_status.as_u16() { + 100..=399 => Some("OK"), + 400..=599 => Some("ERROR"), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn get_header_value_for_span_attribute() { + let expect = "IMPORTANT_HEADER"; + let mut header_map = HeaderMap::new(); + header_map.insert("test", expect.parse().unwrap()); + + let value = get_header_value("test", &header_map); + assert_eq!(value, expect); + } +} diff --git a/reqwest-tracing/src/otel.rs b/reqwest-tracing/src/otel.rs new file mode 100644 index 0000000..aa9c688 --- /dev/null +++ b/reqwest-tracing/src/otel.rs @@ -0,0 +1,62 @@ +use reqwest::header::{HeaderName, HeaderValue}; +use reqwest::Request; +use std::str::FromStr; +use tracing::Span; + +#[cfg(feature = "opentelemetry_0_13")] +use opentelemetry_0_13_pkg as opentelemetry; + +#[cfg(feature = "opentelemetry_0_14")] +use opentelemetry_0_14_pkg as opentelemetry; + +#[cfg(feature = "opentelemetry_0_15")] +use opentelemetry_0_15_pkg as opentelemetry; + +#[cfg(feature = "opentelemetry_0_13")] +pub use tracing_opentelemetry_0_12_pkg as tracing_opentelemetry; + +#[cfg(feature = "opentelemetry_0_14")] +pub use tracing_opentelemetry_0_13_pkg as tracing_opentelemetry; + +#[cfg(feature = "opentelemetry_0_15")] +pub use tracing_opentelemetry_0_14_pkg as tracing_opentelemetry; + +use opentelemetry::global; +use opentelemetry::propagation::Injector; +use tracing_opentelemetry::OpenTelemetrySpanExt; + +/// Injects the given Opentelemetry Context into a reqwest::Request headers to allow propagation downstream. +pub fn inject_opentracing_context_into_request(span: &Span, request: Request) -> Request { + let context = span.context(); + let mut request = request; + + global::get_text_map_propagator(|injector| { + injector.inject_context(&context, &mut RequestCarrier::new(&mut request)) + }); + + request +} + +// "traceparent" => https://www.w3.org/TR/trace-context/#trace-context-http-headers-format + +/// Injector used via opentelemetry propagator to tell the extractor how to insert the "traceparent" header value +/// This will allow the propagator to inject opentelemetry context into a standard data structure. Will basically +/// insert a "traceparent" string value "{version}-{trace_id}-{span_id}-{trace-flags}" of the spans context into the headers. +/// Listeners can then re-hydrate the context to add additional spans to the same trace. +struct RequestCarrier<'a> { + request: &'a mut Request, +} + +impl<'a> RequestCarrier<'a> { + pub fn new(request: &'a mut Request) -> Self { + RequestCarrier { request } + } +} + +impl<'a> Injector for RequestCarrier<'a> { + fn set(&mut self, key: &str, value: String) { + let header_name = HeaderName::from_str(key).expect("Must be header name"); + let header_value = HeaderValue::from_str(&value).expect("Must be a header value"); + self.request.headers_mut().insert(header_name, header_value); + } +}