Skip to content

Commit 9d953d3

Browse files
eifrah-awseifrah-aws
authored
Make redis-rs part of this repo (valkey-io#2456)
* Make redis-rs part of this repo * Improved PYTHON DEVELOPERS.md file * Added Makefile for unified build method for all the languages
1 parent 3661f87 commit 9d953d3

File tree

117 files changed

+43683
-101
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

117 files changed

+43683
-101
lines changed

‎.github/workflows/python.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -216,9 +216,9 @@ jobs:
216216

217217
- name: Install dependencies
218218
if: always()
219-
uses: threeal/pipx-install-action@latest
220-
with:
221-
packages: flake8 isort black
219+
working-directory: ./python
220+
run: |
221+
sudo apt install -y python3-pip python3 flake8 isort black
222222
223223
- name: Lint python with isort
224224
if: always()

‎.github/workflows/semgrep.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,4 @@ jobs:
3333
# Fetch project source with GitHub Actions Checkout.
3434
- uses: actions/checkout@v3
3535
# Run the "semgrep ci" command on the command line of the docker image.
36-
- run: semgrep ci --config auto --no-suppress-errors
36+
- run: semgrep ci --config auto --no-suppress-errors --exclude-rule generic.secrets.security.detected-private-key.detected-private-key

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ logger-rs.linux-x64-gnu.node
4343
utils/clusters/
4444
utils/tls_crts/
4545
utils/TestUtils.js
46+
.build/
47+
.project
4648

4749
# OSS Review Toolkit (ORT) files
4850
**/ort*/**

‎.gitmodules

-3
Original file line numberDiff line numberDiff line change
@@ -1,3 +0,0 @@
1-
[submodule "submodules/redis-rs"]
2-
path = submodules/redis-rs
3-
url = https://github.com/amazon-contributing/redis-rs

‎Makefile

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
.PHONY: all java java-test python python-test node node-test check-redis-server go go-test
2+
3+
BLUE=\033[34m
4+
YELLOW=\033[33m
5+
GREEN=\033[32m
6+
RESET=\033[0m
7+
ROOT_DIR=$(shell pwd)
8+
PYENV_DIR=$(shell pwd)/python/.env
9+
PY_PATH=$(shell find python/.env -name "site-packages"|xargs readlink -f)
10+
PY_GLIDE_PATH=$(shell pwd)/python/python/
11+
12+
all: java java-test python python-test node node-test go go-test python-lint java-lint
13+
14+
##
15+
## Java targets
16+
##
17+
java:
18+
@echo "$(GREEN)Building for Java (release)$(RESET)"
19+
@cd java && ./gradlew :client:buildAllRelease
20+
21+
java-lint:
22+
@echo "$(GREEN)Running spotlessCheck$(RESET)"
23+
@cd java && ./gradlew :spotlessCheck
24+
@echo "$(GREEN)Running spotlessApply$(RESET)"
25+
@cd java && ./gradlew :spotlessApply
26+
27+
java-test: check-redis-server
28+
@echo "$(GREEN)Running integration tests$(RESET)"
29+
@cd java && ./gradlew :integTest:test
30+
31+
##
32+
## Python targets
33+
##
34+
python: .build/python_deps
35+
@echo "$(GREEN)Building for Python (release)$(RESET)"
36+
@cd python && VIRTUAL_ENV=$(PYENV_DIR) .env/bin/maturin develop --release --strip
37+
38+
python-lint: .build/python_deps
39+
@echo "$(GREEN)Building Linters for python$(RESET)"
40+
cd python && \
41+
export VIRTUAL_ENV=$(PYENV_DIR); \
42+
export PYTHONPATH=$(PY_PATH):$(PY_GLIDE_PATH); \
43+
export PATH=$(PYENV_DIR)/bin:$(PATH); \
44+
isort . --profile black --skip-glob python/glide/protobuf --skip-glob .env && \
45+
black . --exclude python/glide/protobuf --exclude .env && \
46+
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics \
47+
--exclude=python/glide/protobuf,.env/* --extend-ignore=E230 && \
48+
flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 \
49+
--statistics --exclude=python/glide/protobuf,.env/* \
50+
--extend-ignore=E230
51+
52+
python-test: .build/python_deps check-redis-server
53+
cd python && PYTHONPATH=$(PY_PATH):$(PY_GLIDE_PATH) .env/bin/pytest --asyncio-mode=auto
54+
55+
.build/python_deps:
56+
@echo "$(GREEN)Generating protobuf files...$(RESET)"
57+
@protoc -Iprotobuf=$(ROOT_DIR)/glide-core/src/protobuf/ \
58+
--python_out=$(ROOT_DIR)/python/python/glide $(ROOT_DIR)/glide-core/src/protobuf/*.proto
59+
@echo "$(GREEN)Building environment...$(RESET)"
60+
@cd python && python3 -m venv .env
61+
@echo "$(GREEN)Installing requirements...$(RESET)"
62+
@cd python && .env/bin/pip install -r requirements.txt
63+
@cd python && .env/bin/pip install -r dev_requirements.txt
64+
@mkdir -p .build/ && touch .build/python_deps
65+
66+
##
67+
## NodeJS targets
68+
##
69+
node: .build/node_deps
70+
@echo "$(GREEN)Building for NodeJS (release)...$(RESET)"
71+
@cd node && npm run build:release
72+
73+
.build/node_deps:
74+
@echo "$(GREEN)Installing NodeJS dependencies...$(RESET)"
75+
@cd node && npm i
76+
@cd node/rust-client && npm i
77+
@mkdir -p .build/ && touch .build/node_deps
78+
79+
node-test: .build/node_deps check-redis-server
80+
@echo "$(GREEN)Running tests for NodeJS$(RESET)"
81+
@cd node && npm run build
82+
cd node && npm test
83+
84+
node-lint: .build/node_deps
85+
@echo "$(GREEN)Running linters for NodeJS$(RESET)"
86+
@cd node && npx run lint:fix
87+
88+
##
89+
## Go targets
90+
##
91+
92+
93+
go: .build/go_deps
94+
$(MAKE) -C go build
95+
96+
go-test: .build/go_deps
97+
$(MAKE) -C go test
98+
99+
go-lint: .build/go_deps
100+
$(MAKE) -C go lint
101+
102+
.build/go_deps:
103+
@echo "$(GREEN)Installing GO dependencies...$(RESET)"
104+
$(MAKE) -C go install-build-tools install-dev-tools
105+
@mkdir -p .build/ && touch .build/go_deps
106+
107+
##
108+
## Common targets
109+
##
110+
check-redis-server:
111+
which redis-server
112+
113+
clean:
114+
rm -fr .build/
115+
116+
help:
117+
@echo "$(GREEN)Listing Makefile targets:$(RESET)"
118+
@echo $(shell grep '^[^#[:space:]].*:' Makefile|cut -d":" -f1|grep -v PHONY|grep -v "^.build"|sort)

‎benchmarks/rust/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ authors = ["Valkey GLIDE Maintainers"]
1111
tokio = { version = "1", features = ["macros", "time", "rt-multi-thread"] }
1212
glide-core = { path = "../../glide-core" }
1313
logger_core = {path = "../../logger_core"}
14-
redis = { path = "../../submodules/redis-rs/redis", features = ["aio"] }
14+
redis = { path = "../../glide-core/redis-rs/redis", features = ["aio"] }
1515
futures = "0.3.28"
1616
rand = "0.8.5"
1717
itoa = "1.0.6"

‎csharp/lib/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ name = "glide_rs"
1212
crate-type = ["cdylib"]
1313

1414
[dependencies]
15-
redis = { path = "../../submodules/redis-rs/redis", features = ["aio", "tokio-comp","tokio-native-tls-comp"] }
15+
redis = { path = "../../glide-core/redis-rs/redis", features = ["aio", "tokio-comp","tokio-native-tls-comp"] }
1616
glide-core = { path = "../../glide-core" }
1717
tokio = { version = "^1", features = ["rt", "macros", "rt-multi-thread", "time"] }
1818
logger_core = {path = "../../logger_core"}

‎glide-core/Cargo.toml

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ authors = ["Valkey GLIDE Maintainers"]
1010
[dependencies]
1111
bytes = "1"
1212
futures = "^0.3"
13-
redis = { path = "../submodules/redis-rs/redis", features = ["aio", "tokio-comp", "tokio-rustls-comp", "connection-manager","cluster", "cluster-async"] }
13+
redis = { path = "./redis-rs/redis", features = ["aio", "tokio-comp", "tokio-rustls-comp", "connection-manager","cluster", "cluster-async"] }
1414
tokio = { version = "1", features = ["macros", "time"] }
1515
logger_core = {path = "../logger_core"}
1616
dispose = "0.5.0"
@@ -42,7 +42,7 @@ serial_test = "3"
4242
criterion = { version = "^0.5", features = ["html_reports", "async_tokio"] }
4343
which = "5"
4444
ctor = "0.2.2"
45-
redis = { path = "../submodules/redis-rs/redis", features = ["tls-rustls-insecure"] }
45+
redis = { path = "./redis-rs/redis", features = ["tls-rustls-insecure"] }
4646
iai-callgrind = "0.9"
4747
tokio = { version = "1", features = ["rt-multi-thread"] }
4848
glide-core = { path = ".", features = ["socket-layer"] } # always enable this feature in tests.

‎glide-core/redis-rs/Cargo.toml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[workspace]
2+
members = ["redis", "redis-test"]
3+
resolver = "2"

‎glide-core/redis-rs/LICENSE

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
Copyright (c) 2022 by redis-rs contributors
2+
3+
Redis cluster code in parts copyright (c) 2018 by Atsushi Koge.
4+
5+
Some rights reserved.
6+
7+
Redistribution and use in source and binary forms, with or without
8+
modification, are permitted provided that the following conditions are
9+
met:
10+
11+
* Redistributions of source code must retain the above copyright
12+
notice, this list of conditions and the following disclaimer.
13+
14+
* Redistributions in binary form must reproduce the above
15+
copyright notice, this list of conditions and the following
16+
disclaimer in the documentation and/or other materials provided
17+
with the distribution.
18+
19+
* The names of the contributors may not be used to endorse or
20+
promote products derived from this software without specific
21+
prior written permission.
22+
23+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
29+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

‎glide-core/redis-rs/Makefile

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
build:
2+
@cargo build
3+
4+
test:
5+
@echo "===================================================================="
6+
@echo "Build all features with lock file"
7+
@echo "===================================================================="
8+
@RUSTFLAGS="-D warnings" cargo build --locked --all-features
9+
10+
@echo "===================================================================="
11+
@echo "Testing Connection Type TCP without features"
12+
@echo "===================================================================="
13+
@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 cargo test --locked -p redis --no-default-features -- --nocapture --test-threads=1 --skip test_module
14+
15+
@echo "===================================================================="
16+
@echo "Testing Connection Type TCP with all features and RESP2"
17+
@echo "===================================================================="
18+
@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 cargo test --locked -p redis --all-features -- --nocapture --test-threads=1 --skip test_module
19+
20+
@echo "===================================================================="
21+
@echo "Testing Connection Type TCP with all features and RESP3"
22+
@echo "===================================================================="
23+
@REDISRS_SERVER_TYPE=tcp PROTOCOL=RESP3 cargo test -p redis --all-features -- --nocapture --test-threads=1 --skip test_module
24+
25+
@echo "===================================================================="
26+
@echo "Testing Connection Type TCP with all features and Rustls support"
27+
@echo "===================================================================="
28+
@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp+tls RUST_BACKTRACE=1 cargo test --locked -p redis --all-features -- --nocapture --test-threads=1 --skip test_module
29+
30+
@echo "===================================================================="
31+
@echo "Testing Connection Type TCP with all features and native-TLS support"
32+
@echo "===================================================================="
33+
@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp+tls RUST_BACKTRACE=1 cargo test --locked -p redis --features=json,tokio-native-tls-comp,connection-manager,cluster-async -- --nocapture --test-threads=1 --skip test_module
34+
35+
@echo "===================================================================="
36+
@echo "Testing Connection Type UNIX"
37+
@echo "===================================================================="
38+
@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=unix RUST_BACKTRACE=1 cargo test --locked -p redis --test parser --test test_basic --test test_types --all-features -- --test-threads=1 --skip test_module
39+
40+
@echo "===================================================================="
41+
@echo "Testing Connection Type UNIX SOCKETS"
42+
@echo "===================================================================="
43+
@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=unix RUST_BACKTRACE=1 cargo test --locked -p redis --all-features -- --test-threads=1 --skip test_cluster --skip test_async_cluster --skip test_module --skip test_cluster_scan
44+
45+
@echo "===================================================================="
46+
@echo "Testing async-std with Rustls"
47+
@echo "===================================================================="
48+
@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 cargo test --locked -p redis --features=async-std-rustls-comp,cluster-async -- --nocapture --test-threads=1 --skip test_module
49+
50+
@echo "===================================================================="
51+
@echo "Testing async-std with native-TLS"
52+
@echo "===================================================================="
53+
@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 cargo test --locked -p redis --features=async-std-native-tls-comp,cluster-async -- --nocapture --test-threads=1 --skip test_module
54+
55+
@echo "===================================================================="
56+
@echo "Testing redis-test"
57+
@echo "===================================================================="
58+
@RUSTFLAGS="-D warnings" RUST_BACKTRACE=1 cargo test --locked -p redis-test
59+
60+
61+
test-module:
62+
@echo "===================================================================="
63+
@echo "Testing RESP2 with module support enabled (currently only RedisJSON)"
64+
@echo "===================================================================="
65+
@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 cargo test --locked --all-features test_module -- --test-threads=1
66+
67+
@echo "===================================================================="
68+
@echo "Testing RESP3 with module support enabled (currently only RedisJSON)"
69+
@echo "===================================================================="
70+
@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 RESP3=true cargo test --all-features test_module -- --test-threads=1
71+
72+
test-single: test
73+
74+
bench:
75+
cargo bench --all-features
76+
77+
docs:
78+
@RUSTFLAGS="-D warnings" RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --all-features --no-deps
79+
80+
upload-docs: docs
81+
@./upload-docs.sh
82+
83+
style-check:
84+
@rustup component add rustfmt 2> /dev/null
85+
cargo fmt --all -- --check
86+
87+
lint:
88+
@rustup component add clippy 2> /dev/null
89+
cargo clippy --all-features --all --tests --examples -- -D clippy::all -D warnings
90+
91+
fuzz:
92+
cd afl/parser/ && \
93+
cargo afl build --bin fuzz-target && \
94+
cargo afl fuzz -i in -o out target/debug/fuzz-target
95+
96+
.PHONY: build test bench docs upload-docs style-check lint fuzz

0 commit comments

Comments
 (0)