diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..f6d6051 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[bumpversion] +files = setup.py +commit = True +tag = True +current_version = 2.0.1rc1 + diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..d6a7f99 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,10 @@ +[run] +source = rocketmq + +[report] +exclude_lines = + pragma: no cover + def __repr__ + raise AssertionError + raise NotImplementedError + if __name__ == .__main__.: diff --git a/.gitignore b/.gitignore index 72dcbef..9a01d77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,77 @@ -.idea/ -cmake-build-*/ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] -*.pyc +# C extensions *.so +# Distribution / packaging +.Python +env/ bin/ -.vscode/ \ No newline at end of file +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +include/ +man/ +etc/ +share/ +*.egg-info/ +.installed.cfg +*.egg + +# pyenv +.python-version + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +pip-selfcheck.json + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage* +.cache +nosetests.xml +coverage.xml + +# Profiling +prof/ + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Sphinx documentation +docs/_build/ + +# PyCharm +.idea/ + +# Vagrant +.vagrant/ + +.mypy_cache/ + +# dylib +rocketmq/*.dylib +rocketmq/*.so +rocketmq/*.dll diff --git a/.travis.yml b/.travis.yml index 786e146..3108b88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,61 @@ -language: cpp sudo: required -dist: trusty -before_install: -- sudo apt-get update -- sudo apt-get install -y git gcc-4.8 g++-4.8 autoconf cmake libtool wget unzip libbz2-dev zlib1g-dev -- sudo apt-get install -y python-dev - -install: -- sudo sh install_boostpython.sh >> tmp_install_boostpython.txt -- mkdir rocketmqlib -- cd rocketmqlib -- wget https://opensource-rocketmq-client-us.oss-us-west-1.aliyuncs.com/cpp-client/linux/1.2.4/RHEL7.x/librocketmq.tar.gz -- tar -xzf librocketmq.tar.gz -- sudo cp librocketmq.so librocketmq.a /usr/local/lib/ -- sudo cp -r rocketmq /usr/local/include/ -- cd ../ +matrix: + include: + - dist: trusty + language: python + python: 2.7 + services: + - docker + script: + - docker pull apacherocketmq/rocketmq-client-python-ci:latest + - docker run --rm -it -v `pwd`:/io -w /io apacherocketmq/rocketmq-client-python-ci:latest /io/manylinux.sh + - ls dist/ + - sudo rm -rf build *.egg-info + - pip install -Ur dev-requirements.txt + - pip install -e . + - pytest --cov=rocketmq -v tests + - pip install codecov && codecov + # Try to install binary wheel + - pip install --force-reinstall dist/*.whl + - dist: trusty + language: python + python: 3.6 + services: + - docker + script: + - docker pull apacherocketmq/rocketmq-client-python-ci:latest + - docker run --rm -it -v `pwd`:/io -w /io apacherocketmq/rocketmq-client-python-ci:latest /io/manylinux.sh + - ls dist/ + - sudo rm -rf build *.egg-info + - pip install -Ur dev-requirements.txt + - pip install -e . + - pytest --cov=rocketmq -v tests + - pip install codecov && codecov + # Try to install binary wheel + - pip install --force-reinstall dist/*.whl + # Build source distribution + - if [[ "${TRAVIS_TAG:-}" != "" ]]; then sudo python setup.py sdist; fi before_script: -- export LD_LIBRARY_PATH=/usr/local/lib + - if [ "$TRAVIS_OS_NAME" == "osx" ]; then export JAVA_HOME=$(/usr/libexec/java_home); fi + - wget https://archive.apache.org/dist/rocketmq/4.5.2/rocketmq-all-4.5.2-bin-release.zip + - unzip rocketmq-all-4.5.2-bin-release.zip + - cd rocketmq-all-4.5.2-bin-release + - perl -i -pe's/-Xms8g -Xmx8g -Xmn4g/-Xms2g -Xmx2g -Xmn1g/g' bin/runbroker.sh + - nohup sh bin/mqnamesrv & + - nohup sh bin/mqbroker -n localhost:9876 & + - sleep 10 + - ./bin/mqadmin updateTopic -b '127.0.0.1:10911' –n '127.0.0.1:9876' -t test + - ./bin/mqadmin updateSubGroup -b '127.0.0.1:10911' –n '127.0.0.1:9876' -g testGroup + - cd .. + +after_failure: + - cat ~/logs/rocketmq-cpp/*.log.* -script: -- mkdir build && cd build -- cmake ../ -DBoost_USE_STATIC_LIBS=OFF -DROCKETMQ_USE_STATIC_LIBS=OFF -- make +after_success: + - | + if [[ "${TRAVIS_TAG:-}" != "" && $TRAVIS_PYTHON_VERSION != "2.7" ]]; then + sudo python -m pip install -U twine; + twine upload --skip-existing dist/*; + fi diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 39d0766..0000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,219 +0,0 @@ -#/* -#* Licensed to the Apache Software Foundation (ASF) under one or more -#* contributor license agreements. See the NOTICE file distributed with -#* this work for additional information regarding copyright ownership. -#* The ASF licenses this file to You under the Apache License, Version 2.0 -#* (the "License"); you may not use this file except in compliance with -#* the License. You may obtain a copy of the License at -#* -#* http://www.apache.org/licenses/LICENSE-2.0 -#* -#* Unless required by applicable law or agreed to in writing, software -#* distributed under the License is distributed on an "AS IS" BASIS, -#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#* See the License for the specific language governing permissions and -#* limitations under the License. -#*/ - -project(rocketmq-client-python) - -cmake_minimum_required(VERSION 2.6) - -set(CMAKE_MACOSX_RPATH 1) - -# CMake complains if we don't have this. -if (COMMAND cmake_policy) - cmake_policy(SET CMP0003 NEW) -endif () - -# We're escaping quotes in the Windows version number, because -# for some reason CMake won't do it at config version 2.4.7 -# It seems that this restores the newer behaviour where define -# args are not auto-escaped. -if (COMMAND cmake_policy) - cmake_policy(SET CMP0005 NEW) -endif () - -if (POLICY CMP0048) - cmake_policy(SET CMP0048 NEW) -endif () - -if (POLICY CMP0064) - cmake_policy(SET CMP0064 NEW) -endif () -# First, declare project (important for prerequisite checks). -project(rocketmq-client-python) - -set(CMAKE_ALLOW_LOOSE_LOOP_CONSTRUCTS ON) -set(CMAKE_VERBOSE_MAKEFILE 1) -#Path of custom cmake -set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) -# put binaries in a different dir to make them easier to find. -set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) -set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) - -# for unix, put debug files in a separate bin "debug" dir. -# release bin files should stay in the root of the bin dir. -# if (CMAKE_GENERATOR STREQUAL "Unix Makefiles") -# if (CMAKE_BUILD_TYPE STREQUAL Debug) -# set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin/debug) -# set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin/debug) -# endif() -# endif() - -if (NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE "Release") -endif () - -set(CXX_FLAGS - -std=c++11 - -g - -Wall - -Wno-deprecated - -fPIC - -fno-strict-aliasing - -Wno-unused-result - -Wno-unused-local-typedef - # -finline-limit=1000 - # -Wextra - # -pedantic - # -pedantic-errors - # -D_FILE_OFFSET_BITS=64 - # -DVALGRIND - # -DCHECK_PTHREAD_RETURN_VALUE - # -Werror - # -Wconversion - # -Wno-unused-parameter - # -Wunused-but-set-variable - # -Wold-style-cast - # -Woverloaded-virtual - # -Wpointer-arith - # -Wshadow - # -Wwrite-strings - # -Wdeprecated-declarations - # -march=native - # -MMD - # -rdynamic - ) - -if (CMAKE_BUILD_BITS EQUAL 32) - list(APPEND CXX_FLAGS "-m32") -else () #not-condition - list(APPEND CXX_FLAGS "-m64") -endif () - -string(REPLACE ";" " " CMAKE_CXX_FLAGS "${CXX_FLAGS}") -# set(CMAKE_CXX_COMPILER "c++") -set(CMAKE_CXX_FLAGS_DEBUG "-O0 -DDEBUG") -set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") - -include("cmake/ConfigureChecks.cmake") - -# Declare deplibs, so we can use list in linker later. There's probably -# a more elegant way of doing this; with SCons, when you check for the -# lib, it is automatically passed to the linker. -set(deplibs) - -# For some reason, the check_function_exists macro doesn't detect -# the inet_aton on some pure Unix platforms (e.g. sunos5). So we -# need to do a more detailed check and also include some extra deplibs. - -# pthread is used on both Linux and Mac -check_library_exists("pthread" pthread_create "" HAVE_PTHREAD) -if (HAVE_PTHREAD) - list(APPEND deplibs pthread) -else () - message(FATAL_ERROR "Missing library: pthread") -endif () - -check_library_exists(dl dlopen "" HAVE_LIBDL) -if (HAVE_LIBDL) - list(APPEND deplibs dl) -else () #not-HAVE_LIBDL - message(FATAL_ERROR "Missing library: dl") -endif () - -check_library_exists(z compress "" HAVE_LIBZ) -if (HAVE_LIBZ) - list(APPEND deplibs z) -else () #not-HAVE_LIBZ - message(FATAL_ERROR "Missing library: z") -endif () - - -if (WIN32) - find_package(PythonLibs 2.7) -elseif (APPLE) - find_package(PythonLibs 2.7) -else () - find_package(PythonLibs 2.7) -endif (WIN32) - -if (PYTHONLIBS_FOUND) - message(STATUS "** Python Include dir: ${PYTHON_INCLUDE_DIRS}") - message(STATUS "** Python Libraries dir: ${PYTHON_LIBRARY_DIRS}") - message(STATUS "** Python Libraries: ${PYTHON_LIBRARIES}") - include_directories(${PYTHON_INCLUDE_DIRS}) -else () - message(FATAL_ERROR "Missing library: python-devel ") -endif () - -option(Boost_USE_STATIC_LIBS "only find boost static libs" OFF) # find boost libs -if (WIN32) - find_package(Boost REQUIRED COMPONENTS python) -elseif (APPLE) - find_package(Boost REQUIRED COMPONENTS python) -else () - find_package(Boost REQUIRED COMPONENTS python) -endif (WIN32) - -if (Boost_FOUND) - message(STATUS "** Boost Include dir: ${Boost_INCLUDE_DIR}") - message(STATUS "** Boost Libraries dir: ${Boost_LIBRARY_DIRS}") - message(STATUS "** Boost Libraries: ${Boost_LIBRARIES}") - include_directories(${Boost_INCLUDE_DIRS}) -else () - message(FATAL_ERROR "Missing library: boost_python") -endif () - -option(ROCKETMQ_USE_STATIC_LIBS "only find rocketmq static libs" ON) #find rocketmq libs - -find_package(rocketmq) - -if (ROCKETMQ_FOUND) - message(STATUS "** Rocketmq Include dir: ${ROCKETMQ_INCLUDE_DIRS}") - message(STATUS "** Rocketmq Libraries dir: ${ROCKETMQ_LIBRARY_DIRS}") - message(STATUS "** Rocketmq Libraries: ${ROCKETMQ_LIBRARIES}") - include_directories(${ROCKETMQ_INCLUDE_DIRS}) -else () - message(FATAL_ERROR "Missing library: rocketmq") -endif () -# add include dir for bsd (posix uses /usr/include/) -set(CMAKE_INCLUDE_PATH "${CMAKE_INCLUDE_PATH}:/usr/include") - -# For config.h, set some static values; it may be a good idea to make -# these values dynamic for non-standard UNIX compilers. -set(ACCEPT_TYPE_ARG3 socklen_t) -set(HAVE_CXX_BOOL 1) -set(HAVE_CXX_CASTS 1) -set(HAVE_CXX_EXCEPTIONS 1) -set(HAVE_CXX_MUTABLE 1) -set(HAVE_CXX_STDLIB 1) -set(HAVE_PTHREAD_SIGNAL 1) -set(SELECT_TYPE_ARG1 int) -set(SELECT_TYPE_ARG234 "(fd_set *)") -set(SELECT_TYPE_ARG5 "(struct timeval *)") -set(STDC_HEADERS 1) -set(TIME_WITH_SYS_TIME 1) -set(HAVE_SOCKLEN_T 1) - -option(TEST "Build test cases" OFF) -if (TEST) - enable_testing() - option(gtest_build_tests OFF) - #add_subdirectory(third_party/googletest/googletest) - include_directories(SYSTEM ${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR}) - add_subdirectory(unitests) -endif () - -add_subdirectory(project) diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..67afa4c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include README.md rocketmq/librocketmq.dylib rocketmq/librocketmq_client_core.dylib rocketmq/librocketmq.so rocketmq/librocketmq_client_core.so + +recursive-exclude * .DS_Store diff --git a/Makefile b/Makefile deleted file mode 100644 index 6fc2d70..0000000 --- a/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -#/* -#* Licensed to the Apache Software Foundation (ASF) under one or more -#* contributor license agreements. See the NOTICE file distributed with -#* this work for additional information regarding copyright ownership. -#* The ASF licenses this file to You under the Apache License, Version 2.0 -#* (the "License"); you may not use this file except in compliance with -#* the License. You may obtain a copy of the License at -#* -#* http://www.apache.org/licenses/LICENSE-2.0 -#* -#* Unless required by applicable law or agreed to in writing, software -#* distributed under the License is distributed on an "AS IS" BASIS, -#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#* See the License for the specific language governing permissions and -#* limitations under the License. -#*/ - -LIBS_ORIG := $(patsubst %/,%, $(dir $(wildcard libs/*/Makefile))) -LIBS_CLEAN := $(addsuffix -clean,$(LIBS_ORIG)) - -.PHONY: $(LIBS_CLEAN) - -all: build-shared - -build-libs: $(LIBS_ORIG) - -$(LIBS_ORIG): - $(MAKE) -C $@ - -build-shared: - $(MAKE) -C project - -test: - @echo $(LIBS_ORIG) - -# clean:$(LIBS_CLEAN) -clean: - $(MAKE) -C project clean - $(MAKE) -C bin clean - $(RM) -rf logs/*.log - $(RM) -rf tmp - -cleanall:$(LIBS_CLEAN) clean - -install: - $(MAKE) -C project install - -$(LIBS_CLEAN): - $(MAKE) -C $(@:-clean=) clean diff --git a/NOTICE b/NOTICE index 703c28b..65ebdd0 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache RocketMQ -Copyright 2016-2018 The Apache Software Foundation +Copyright 2016-2020 The Apache Software Foundation This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). \ No newline at end of file +The Apache Software Foundation (http://www.apache.org/). diff --git a/README.md b/README.md index 3b1c67b..2140022 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,98 @@ -## RocketMQ Client Python +# rocketmq-client-python + [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) -[![TravisCI](https://travis-ci.org/apache/rocketmq-client-python.svg?branch=master)](https://travis-ci.org/apache/rocketmq-client-python) -* RocketMQ Python client is developed on top of [rocketmq-client-cpp](https://github.com/apache/rocketmq-client-cpp), which has been proven robust and widely adopted within Alibaba Group by many business units for more than three years. - ----------- -## Quick Start -* Step-by-step instruction are provided in [RocketMQ Client Python Introduction](https://github.com/apache/rocketmq-client-python/blob/master/doc/Introduction.md). -* Consult [RocketMQ Quick Start](https://rocketmq.apache.org/docs/quick-start/) to setup rocketmq broker and nameserver. - ----------- -## Apache RocketMQ Community -* [RocketMQ Community Projects](https://github.com/apache/rocketmq-externals) - ----------- -## Contact us -* Mailing Lists: -* Home: -* Docs: -* Issues: -* Ask: -* Slack: - ----------- -## How to Contribute - Contributions are warmly welcome! Be it trivial cleanup, major new feature or other suggestion. Read this [how to contribute](http://rocketmq.apache.org/docs/how-to-contribute/) guide for more details. - - ----------- +[![Build Status](https://travis-ci.org/apache/rocketmq-client-python.svg?branch=master)](https://travis-ci.org/apache/rocketmq-client-python) +[![codecov](https://codecov.io/gh/apache/rocketmq-client-python/branch/ctypes/graph/badge.svg)](https://codecov.io/gh/apache/rocketmq-client-python/branch/ctypes) +[![PyPI](https://img.shields.io/pypi/v/rocketmq-client-python.svg)](https://pypi.org/project/rocketmq-client-python) +[![GitHub release](https://img.shields.io/badge/release-download-default.svg)](https://github.com/apache/rocketmq-client-python/releases) +[![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/apache/rocketmq-client-python.svg)](http://isitmaintained.com/project/apache/rocketmq-client-python "Average time to resolve an issue") +[![Percentage of issues still open](http://isitmaintained.com/badge/open/apache/rocketmq-client-python.svg)](http://isitmaintained.com/project/apache/rocketmq-client-python "Percentage of issues still open") +![Twitter Follow](https://img.shields.io/twitter/follow/ApacheRocketMQ?style=social) + +RocketMQ Python client, based on [rocketmq-client-cpp](https://github.com/apache/rocketmq-client-cpp), supports Linux and macOS +## Prerequisites + +### Install `librocketmq` +rocketmq-client-python is a lightweight wrapper around [rocketmq-client-cpp](https://github.com/apache/rocketmq-client-cpp), so you need install +`librocketmq` first. + +#### Download by binary release. +download specific release according you OS: [rocketmq-client-cpp-2.0.0](https://github.com/apache/rocketmq-client-cpp/releases/tag/2.0.0) +- centos + + take centos7 as example, you can install the library in centos6 by the same method. + ```bash + wget https://github.com/apache/rocketmq-client-cpp/releases/download/2.0.0/rocketmq-client-cpp-2.0.0-centos7.x86_64.rpm + sudo rpm -ivh rocketmq-client-cpp-2.0.0-centos7.x86_64.rpm + ``` +- debian + ```bash + wget https://github.com/apache/rocketmq-client-cpp/releases/download/2.0.0/rocketmq-client-cpp-2.0.0.amd64.deb + sudo dpkg -i rocketmq-client-cpp-2.0.0.amd64.deb + ``` +- macOS + ```bash + wget https://github.com/apache/rocketmq-client-cpp/releases/download/2.0.0/rocketmq-client-cpp-2.0.0-bin-release.darwin.tar.gz + tar -xzf rocketmq-client-cpp-2.0.0-bin-release.darwin.tar.gz + cd rocketmq-client-cpp + mkdir /usr/local/include/rocketmq + cp include/* /usr/local/include/rocketmq + cp lib/* /usr/local/lib + install_name_tool -id "@rpath/librocketmq.dylib" /usr/local/lib/librocketmq.dylib + ``` +#### Build from source +you can also build it manually from source according to [Build and Install](https://github.com/apache/rocketmq-client-cpp/tree/master#build-and-install) + +## Installation + +```bash +pip install rocketmq-client-python +``` + +## Usage + +### Producer + +```python +from rocketmq.client import Producer, Message + +producer = Producer('PID-XXX') +producer.set_name_server_address('127.0.0.1:9876') +producer.start() + +msg = Message('YOUR-TOPIC') +msg.set_keys('XXX') +msg.set_tags('XXX') +msg.set_body('XXXX') +ret = producer.send_sync(msg) +print(ret.status, ret.msg_id, ret.offset) +producer.shutdown() +``` + +### PushConsumer + +```python +import time + +from rocketmq.client import PushConsumer, ConsumeStatus + + +def callback(msg): + print(msg.id, msg.body) + return ConsumeStatus.CONSUME_SUCCESS + + +consumer = PushConsumer('CID_XXX') +consumer.set_name_server_address('127.0.0.1:9876') +consumer.subscribe('YOUR-TOPIC', callback) +consumer.start() + +while True: + time.sleep(3600) + +consumer.shutdown() + +``` + ## License - [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright (C) Apache Software Foundation +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright (C) Apache Software Foundation diff --git a/changelog b/changelog deleted file mode 100755 index e474814..0000000 --- a/changelog +++ /dev/null @@ -1,7 +0,0 @@ -version 1.0.0 @2018.10.16 - * Initialize version of python client. - * Export python APIs Based on CPP SDK using boost-python. - * Support sending message in synchronous mode. - * Support consuming message using push model. - * Support python 2.7.X under linux platform. - diff --git a/ci/Dockerfile b/ci/Dockerfile new file mode 100644 index 0000000..8066756 --- /dev/null +++ b/ci/Dockerfile @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM quay.io/pypa/manylinux1_x86_64:latest + +RUN yum install -y wget curl gcc libtool unzip automake autoconf bzip2-devel && ln -s `which cmake28` /usr/bin/cmake + +# Install zlib +RUN curl -sqL https://zlib.net/zlib-1.2.11.tar.gz | tar -xz -C /tmp && \ + cd /tmp/zlib-1.2.11/ && \ + ./configure --prefix=/usr && \ + make && \ + make install && \ + cd - && \ + rm -rf /tmp/zlib-1.2.11 + +# Build rocketmq-client-cpp +RUN git clone --depth=1 --branch=master https://github.com/apache/rocketmq-client-cpp.git /tmp/rocketmq-client-cpp && \ + mkdir -p /tmp/rocketmq-client-cpp/tmp_down_dir && \ + curl -sqL -o /tmp/rocketmq-client-cpp/tmp_down_dir/libevent-release-2.1.8-stable.zip https://github.com/libevent/libevent/archive/release-2.1.8-stable.zip && \ + curl -sqL -o /tmp/rocketmq-client-cpp/tmp_down_dir/jsoncpp-0.10.7.zip https://github.com/open-source-parsers/jsoncpp/archive/0.10.7.zip && \ + curl -sqL -o /tmp/rocketmq-client-cpp/tmp_down_dir/boost_1_58_0.tar.gz http://sourceforge.net/projects/boost/files/boost/1.58.0/boost_1_58_0.tar.gz && \ + cd /tmp/rocketmq-client-cpp && bash build.sh && cd - && \ + cp /tmp/rocketmq-client-cpp/bin/librocketmq.so /usr/local/lib/librocketmq.so && \ + rm -rf /tmp/rocketmq-client-cpp diff --git a/cmake/ConfigureChecks.cmake b/cmake/ConfigureChecks.cmake deleted file mode 100644 index c2e0d4a..0000000 --- a/cmake/ConfigureChecks.cmake +++ /dev/null @@ -1,356 +0,0 @@ - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -include(CheckIncludeFiles) -include(CheckFunctionExists) -include(CheckLibraryExists) -include(CheckSymbolExists) -include(CheckTypeSize) -include(CheckCSourceCompiles) -include(CheckCXXSourceCompiles) - -check_include_files(dlfcn.h HAVE_DLFCN_H ) - -check_include_files(errno.h HAVE_ERRNO_H ) -check_include_files(iconv.h HAVE_ICONV_H ) -check_include_files(limits.h HAVE_LIMITS_H ) -check_include_files(sys/types.h HAVE_SYS_TYPES_H ) -check_include_files("sys/types.h;sys/socket.h" HAVE_SYS_SOCKET_H ) -check_include_files(sys/syscall.h HAVE_SYS_SYSCALL_H ) -check_include_files("sys/types.h;sys/time.h" HAVE_SYS_TIME_H ) -check_include_files("sys/types.h;sys/timeb.h" HAVE_SYS_TIMEB_H ) -check_include_files("sys/types.h;sys/stat.h" HAVE_SYS_STAT_H ) -check_include_files(sys/file.h HAVE_SYS_FILE_H ) -check_include_files(syslog.h HAVE_SYSLOG_H ) -check_include_files(arpa/inet.h HAVE_ARPA_INET_H ) -check_include_files(netinet/in.h HAVE_NETINET_IN_H ) -check_include_files("sys/types.h;netinet/tcp.h" HAVE_NETINET_TCP_H ) -check_include_files(netdb.h HAVE_NETDB_H ) -check_include_files(unistd.h HAVE_UNISTD_H ) -check_include_files(fcntl.h HAVE_FCNTL_H ) -check_include_files(stdio.h HAVE_STDIO_H ) -check_include_files(stdarg.h HAVE_STDARG_H ) -check_include_files(stdlib.h HAVE_STDLIB_H ) -check_include_files(time.h HAVE_TIME_H ) -check_include_files(wchar.h HAVE_WCHAR_H ) -check_include_files(poll.h HAVE_POLL_H ) - - -check_include_files(inttypes.h HAVE_INTTYPES_H ) -check_include_files(memory.h HAVE_MEMORY_H ) -check_include_files(stdint.h HAVE_STDINT_H ) -check_include_files(strings.h HAVE_STRINGS_H ) -check_include_files(string.h HAVE_STRING_H ) - - -check_include_files("stdlib.h;stdio.h;stdarg.h;string.h;float.h" STDC_HEADERS ) - -find_library(LIBADVAPI32 advapi32) -find_library(LIBKERNEL32 kernel32) -find_library(LIBNSL nsl) -find_library(LIBRT rt) -find_library(LIBICONV iconv) -find_library(LIBPOSIX4 posix4) -find_library(LIBCPOSIX cposix) -find_library(LIBSOCKET socket) -find_library(LIBWS2_32 ws2_32) - -check_function_exists(gmtime_r HAVE_GMTIME_R ) -check_function_exists(localtime_r HAVE_LOCALTIME_R ) -check_function_exists(gettimeofday HAVE_GETTIMEOFDAY ) -check_function_exists(getpid HAVE_GETPID ) -check_function_exists(poll HAVE_POLL ) -check_function_exists(pipe HAVE_PIPE ) -check_function_exists(pipe2 HAVE_PIPE2 ) -check_function_exists(ftime HAVE_FTIME ) -check_function_exists(stat HAVE_STAT ) -check_function_exists(lstat HAVE_LSTAT ) -check_function_exists(fcntl HAVE_FCNTL ) -check_function_exists(lockf HAVE_FLOCK ) -check_function_exists(flock HAVE_LOCKF ) -check_function_exists(htons HAVE_HTONS ) -check_function_exists(ntohs HAVE_NTOHS ) -check_function_exists(htonl HAVE_HTONL ) -check_function_exists(ntohl HAVE_NTOHL ) -check_function_exists(shutdown HAVE_SHUTDOWN ) -check_function_exists(vsnprintf HAVE_VSNPRINTF ) -check_function_exists(_vsnprintf HAVE__VSNPRINTF ) -check_function_exists(vsprintf_s HAVE_VSPRINTF_S ) -check_function_exists(vswprintf_s HAVE_VSWPRINTF_S ) -check_function_exists(vfprintf_s HAVE_VFPRINTF_S ) -check_function_exists(vfwprintf_s HAVE_VFWPRINTF_S ) -check_function_exists(_vsnprintf_s HAVE__VSNPRINTF_S ) -check_function_exists(_vsnwprintf_s HAVE__VSNWPRINTF_S ) -check_function_exists(mbstowcs HAVE_MBSTOWCS ) -check_function_exists(wcstombs HAVE_WCSTOMBS ) - - -check_symbol_exists(ENAMETOOLONG errno.h HAVE_ENAMETOOLONG ) -check_symbol_exists(SYS_gettid sys/syscall.h HAVE_GETTID ) -check_symbol_exists(__FUNCTION__ "" HAVE_FUNCTION_MACRO ) -check_symbol_exists(__PRETTY_FUNCTION__ "" HAVE_PRETTY_FUNCTION_MACRO ) -check_symbol_exists(__func__ "" HAVE_FUNC_SYMBOL ) - -check_c_source_compiles("#include \n int main() { int x = 1; int y = __sync_add_and_fetch (&x, 1); return y;}" - HAVE___SYNC_ADD_AND_FETCH ) - -check_c_source_compiles("#include \n int main() { int x = 1; int y = __sync_sub_and_fetch (&x, 1); return y;}" - HAVE___SYNC_SUB_AND_FETCH ) - -check_c_source_compiles("#include \n #define MACRO(buf, args...) (sprintf (buf, \"%d\", args))\n int main() {char a[10]; MACRO(a, 1); return 0; }" - HAVE_GNU_VARIADIC_MACROS ) - -check_c_source_compiles("#include \n #define MACRO(buf, ...) (sprintf (buf, \"%d\", __VA_ARGS__))\n int main() {char a[10]; MACRO(a, 1); return 0; }" - HAVE_C99_VARIADIC_MACROS ) - - -# clock_gettime() needs -lrt here -# TODO AC says this exists -if (LIBRT) - check_library_exists("${LIBRT}" clock_gettime "" - HAVE_CLOCK_GETTIME ) - check_library_exists("${LIBRT}" clock_nanosleep "" - HAVE_CLOCK_NANOSLEEP ) - check_library_exists("${LIBRT}" nanosleep "" - HAVE_NANOSLEEP ) -else () - check_function_exists(clock_gettime HAVE_CLOCK_GETTIME ) - check_function_exists(clock_nanosleep HAVE_CLOCK_NANOSLEEP ) - check_function_exists(nanosleep HAVE_NANOSLEEP ) -endif () - -# iconv functions may require iconv library (on OS X for example) -if(WITH_ICONV) - if(LIBICONV) - check_library_exists("${LIBICONV}" iconv_open "" HAVE_ICONV_OPEN ) - check_library_exists("${LIBICONV}" iconv_close "" HAVE_ICONV_CLOSE ) - check_library_exists("${LIBICONV}" iconv "" HAVE_ICONV ) - else() - check_function_exists(iconv_open HAVE_ICONV_OPEN ) - check_function_exists(iconv_close HAVE_ICONV_CLOSE ) - check_function_exists(iconv HAVE_ICONV ) - endif() -endif() - -check_function_exists(gethostbyname_r HAVE_GETHOSTBYNAME_R) # TODO more complicated test in AC -check_function_exists(getaddrinfo HAVE_GETADDRINFO ) # TODO more complicated test in AC - - -# check for declspec stuff -if(NOT DEFINED LOG4CPLUS_DECLSPEC_EXPORT) - check_c_source_compiles( - "#if defined (__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ <= 1)) - # error Please fail. - #endif - - __attribute__((visibility(\"default\"))) int x = 0; - __attribute__((visibility(\"default\"))) int foo(); - int foo() { return 0; } - __attribute__((visibility(\"default\"))) int bar() { return x; } - __attribute__((visibility(\"hidden\"))) int baz() { return 1; } - - int main(void) { return 0; }" - HAVE_ATTRIBUTE_VISIBILITY - ) - if(HAVE_ATTRIBUTE_VISIBILITY) - set(LOG4CPLUS_DECLSPEC_EXPORT "__attribute__ ((visibility(\"default\")))" ) - set(LOG4CPLUS_DECLSPEC_IMPORT "__attribute__ ((visibility(\"default\")))" ) - set(LOG4CPLUS_DECLSPEC_PRIVATE "__attribute__ ((visibility(\"hidden\")))" ) - endif() -endif() - -if(NOT DEFINED LOG4CPLUS_DECLSPEC_EXPORT) - check_c_source_compiles( - "#if defined (__clang__) - // Here the problem is that Clang only warns that it does not support - // __declspec(dllexport) but still compiles the executable. - # error Please fail. - #endif - - __declspec(dllexport) int x = 0; - __declspec(dllexport) int foo (); - int foo () { return 0; } - __declspec(dllexport) int bar () { return x; } - - int main(void) { return 0; }" - HAVE_DECLSPEC_DLLEXPORT - ) - if(HAVE_DECLSPEC_DLLEXPORT) - set(LOG4CPLUS_DECLSPEC_EXPORT "__declspec(dllexport)" ) - set(LOG4CPLUS_DECLSPEC_IMPORT "__declspec(dllimport)" ) - set(LOG4CPLUS_DECLSPEC_PRIVATE "" ) - endif() -endif() - -if(NOT DEFINED LOG4CPLUS_DECLSPEC_EXPORT) - check_c_source_compiles( - "__global int x = 0; - __global int foo(); - int foo() { return 0; } - __global int bar() { return x; } - __hidden int baz() { return 1; } - - int main(void) { return 0; }" - HAVE_GLOBAL_AND_HIDDEN - ) - if(HAVE_GLOBAL_AND_HIDDEN) - set(LOG4CPLUS_DECLSPEC_EXPORT "__global" ) - set(LOG4CPLUS_DECLSPEC_IMPORT "__global" ) - set(LOG4CPLUS_DECLSPEC_PRIVATE "__hidden" ) - endif() -endif() - -if(NOT DEFINED LOG4CPLUS_DECLSPEC_EXPORT OR NOT ENABLE_SYMBOLS_VISIBILITY) - set(LOG4CPLUS_DECLSPEC_EXPORT "") - set(LOG4CPLUS_DECLSPEC_IMPORT "") - set(LOG4CPLUS_DECLSPEC_PRIVATE "") -endif() - -# check for thread-local stuff -if(NOT DEFINED LOG4CPLUS_HAVE_TLS_SUPPORT) - # TODO: requires special compiler switch on GCC and Clang - # Currently it is assumed that they are provided in - # CMAKE_CXX_FLAGS - set(CMAKE_REQUIRED_FLAGS "${CMAKE_CXX_FLAGS}") - check_cxx_source_compiles( - "extern thread_local int x; - thread_local int * ptr = 0; - int foo() { ptr = &x; return x; } - thread_local int x = 1; - - int main() - { - x = 2; - foo(); - return 0; - }" - HAVE_CXX11_THREAD_LOCAL - ) - set(CMAKE_REQUIRED_FLAGS "") - if(HAVE_CXX11_THREAD_LOCAL) - set(LOG4CPLUS_HAVE_TLS_SUPPORT 1) - set(LOG4CPLUS_THREAD_LOCAL_VAR "thread_local") - endif() -endif() - -if(NOT DEFINED LOG4CPLUS_HAVE_TLS_SUPPORT) - check_cxx_source_compiles( - "#if defined (__NetBSD__) - #include - #if ! __NetBSD_Prereq__(5,1,0) - #error NetBSD __thread support does not work before 5.1.0. It is missing __tls_get_addr. - #endif - #endif - - extern __thread int x; - __thread int * ptr = 0; - int foo() { ptr = &x; return x; } - __thread int x = 1; - - int main() - { - x = 2; - foo(); - return 0; - }" - HAVE_GCC_THREAD_EXTENSION - ) - if(HAVE_GCC_THREAD_EXTENSION) - set(LOG4CPLUS_HAVE_TLS_SUPPORT 1) - set(LOG4CPLUS_THREAD_LOCAL_VAR "__thread") - endif() -endif() - -if(NOT DEFINED LOG4CPLUS_HAVE_TLS_SUPPORT) - check_cxx_source_compiles( - "#if defined (__GNUC__) - #error Please fail. - #endif - - extern __declspec(thread) int x; - __declspec(thread) int * ptr = 0; - int foo() { ptr = &x; return x; } - __declspec(thread) int x = 1; - - int main() - { - x = 2; - foo(); - return 0; - }" - HAVE_DECLSPEC_THREAD - ) - if(HAVE_DECLSPEC_THREAD) - set(LOG4CPLUS_HAVE_TLS_SUPPORT 1) - set(LOG4CPLUS_THREAD_LOCAL_VAR "__declspec(thread)") - endif() -endif() - -# check for c++11 atomic stuff -# TODO: requires special compiler switch on GCC and Clang -# Currently it is assumed that they are provided in -# CMAKE_CXX_FLAGS -set(CMAKE_REQUIRED_FLAGS "${CMAKE_CXX_FLAGS}") -check_cxx_source_compiles( - "#include - - template - void test_atomic() - { - std::atomic x(0); - std::atomic_fetch_add_explicit(&x, static_cast(1), std::memory_order_acquire); - std::atomic_fetch_sub_explicit(&x, static_cast(1), std::memory_order_release); - } - - int main() - { - test_atomic(); - test_atomic(); - test_atomic(); - test_atomic(); - std::atomic_thread_fence(std::memory_order_acquire); - return 0; - }" - LOG4CPLUS_HAVE_CXX11_ATOMICS -) -set(CMAKE_REQUIRED_FLAGS "") - -set(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h) -check_type_size(socklen_t _SOCKLEN_SIZE) -if (_SOCKLEN_SIZE) - set(socklen_t) -else() - set(socklen_t TRUE) -endif() - -macro(PATH_TO_HAVE _pathVar ) - if (${_pathVar}) - set(HAVE_${_pathVar} TRUE) - else () - set(HAVE_${_pathVar} FALSE) - endif () -endmacro() - - -path_to_have(LIBADVAPI32) -path_to_have(LIBKERNEL32) -path_to_have(LIBPOSIX4) -path_to_have(LIBCPOSIX) -path_to_have(LIBSOCKET) -path_to_have(LIBWS2_32) - - - diff --git a/cmake/Findrocketmq.cmake b/cmake/Findrocketmq.cmake deleted file mode 100644 index 5df1bbf..0000000 --- a/cmake/Findrocketmq.cmake +++ /dev/null @@ -1,115 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Find RocketMQ Libary -# -# Find the rocketmq includes and library -# -# if you need to add a custom library search path, do it via CMAKE_PREFIX_PATH -# -# -*- cmake -*- -# - Find Rocketmq -# Find the Rocketmq includes and library -# This module defines -# ROCKETMQ_INCLUDE_DIRS, where to find CCommon.h, CMessage.h, etc.#include "CCommon.h" -# ROCKETMQ_LIBRARIES, the libraries needed to use Rocketmq. -# ROCKETMQ_FOUND, If false, do not try to use Rocketmq. -# also defined, but not for general use are -# ROCKETMQ_LIBRARIES, where to find the rocketmq library. - -# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES -if (ROCKETMQ_USE_STATIC_LIBS) - set(_rocketmq_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES :${CMAKE_FIND_LIBRARY_SUFFIXES}) - if (WIN32) - list(INSERT CMAKE_FIND_LIBRARY_SUFFIXES 0 .lib .a) - else () - set(CMAKE_FIND_LIBRARY_SUFFIXES .a) - endif () -else () - set(_rocketmq_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES :${CMAKE_FIND_LIBRARY_SUFFIXES}) - if (WIN32) - list(INSERT CMAKE_FIND_LIBRARY_SUFFIXES 0 .dll) - elseif (APPLE) - set(CMAKE_FIND_LIBRARY_SUFFIXES .dylib) - else () - set(CMAKE_FIND_LIBRARY_SUFFIXES .so) - endif (WIN32) -endif (ROCKETMQ_USE_STATIC_LIBS) -if (ROCKETMQ_USE_SHARED_LIBS) - set(_rocketmq_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES :${CMAKE_FIND_LIBRARY_SUFFIXES}) - if (WIN32) - list(INSERT CMAKE_FIND_LIBRARY_SUFFIXES 0 .dll) - elseif (APPLE) - set(CMAKE_FIND_LIBRARY_SUFFIXES .dylib) - else () - set(CMAKE_FIND_LIBRARY_SUFFIXES .so) - endif () -endif (ROCKETMQ_USE_SHARED_LIBS) - -FIND_PATH(ROCKETMQ_INCLUDE_DIRS - NAMES - CCommon.h - PATHS - /usr/include - /usr/local/include - C:/rocketmq/include - ${CMAKE_SOURCE_DIR}/win32-deps/include - ${ROCKETMQ_LIBRARY_DIRS} - PATH_SUFFIXES rocketmq - ) -message(STATUS "***rocketmq include path: ${ROCKETMQ_INCLUDE_DIRS}") - -find_library(ROCKETMQ_LIBRARIES - NAMES rocketmq - PATHS - /usr/lib - /usr/local/lib - C:/rocketmq/lib - ${CMAKE_SOURCE_DIR}/win32-deps/lib - ${ROCKETMQ_LIBRARY_DIRS} - ) -message(STATUS "***rocketmq libaray: ${ROCKETMQ_LIBRARIES}") - -IF (ROCKETMQ_LIBRARIES AND ROCKETMQ_INCLUDE_DIRS) - SET(ROCKETMQ_LIBRARIES ${ROCKETMQ_LIBRARIES}) - SET(ROCKETMQ_FOUND "YES") - message(STATUS "***Find library: rocketmq") -ELSE (ROCKETMQ_LIBRARIES AND ROCKETMQ_INCLUDE_DIRS) - SET(ROCKETMQ_FOUND "NO") - message(FATAL_ERROR "Missing library: rocketmq") -ENDIF (ROCKETMQ_LIBRARIES AND ROCKETMQ_INCLUDE_DIRS) - - -IF (ROCKETMQ_FOUND) - IF (NOT ROCKETMQ_FIND_QUIETLY) - MESSAGE(STATUS "***Found Rocketmq: ${ROCKETMQ_LIBRARIES}") - ENDIF (NOT ROCKETMQ_FIND_QUIETLY) -ELSE (ROCKETMQ_FOUND) - IF (ROCKETMQ_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find ROCKETMQ library include: ${ROCKETMQ_INCLUDE_DIRS}, lib: ${ROCKETMQ_LIBRARIES}") - ENDIF (ROCKETMQ_FIND_REQUIRED) -ENDIF (ROCKETMQ_FOUND) - -MARK_AS_ADVANCED( - ROCKETMQ_LIBRARIES - ROCKETMQ_INCLUDE_DIRS -) - -# Restore the original find library ordering -if (ROCKETMQ_USE_STATIC_LIBS) - set(CMAKE_FIND_LIBRARY_SUFFIXES ${_rocketmq_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) -else () - set(CMAKE_FIND_LIBRARY_SUFFIXES ${_rocketmq_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) -endif () diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..c96c923 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pytest +pytest-timeout +pytest-faulthandler +pytest-cov +futures; python_version < '3' diff --git a/doc/Introduction.md b/doc/Introduction.md deleted file mode 100644 index c0f97da..0000000 --- a/doc/Introduction.md +++ /dev/null @@ -1,161 +0,0 @@ ----------- -## RocketMQ Client Python - -### 1. Python Runtime Version -* python 2.7.x - - -### 2. Dependency of Python Client - -* CPP Core: [librocketmq](https://github.com/apache/rocketmq-client-cpp) -* python-devel 2.7.x -* boost-python 1.58.0 - -### 3. Build and Install -#### Linux Platform -* Install compile tools: - ``` - - sudo yum install make - - sudo yum install cmake - - sudo yum install gcc-c++ - ``` -* Install dependency: - - 1. python-devel - ``` - sudo yum install python-devel - ``` - - 2. zlib-devel - ``` - sudo yum install zlib-devel - ``` - 3. boost-python - ``` - sudo sh install_boostpython.sh - ``` - 4. [librocketmq](https://github.com/apache/rocketmq-client-cpp), choose one method below: - - - build from source: [Build and Install](https://github.com/apache/rocketmq-client-cpp/tree/master#build-and-install) - - - download specific release: [rocketmq-client-cpp](https://www.apache.org/dyn/closer.cgi?path=rocketmq/rocketmq-client-cpp/1.2.4/rocketmq-client-cpp-1.2.4-bin-release.tar.gz) and unzip the package, please choose the right version according to your OS and unzip it, then copy the library files to to your `/usr/local/lib/` directory and copy the head files under `include` path to `/usr/local/include/rocketmq/`. and please make sure your `/usr/local/lib/` directory is under the `LD_LIBRARY_PATH`. - -* Make and install module manually - 1. Using Dynamic RocketMQ and boost python libraries are recommended. - ``` - - mkdir build && cd build - - cmake ../ -DBoost_USE_STATIC_LIBS=OFF -DROCKETMQ_USE_STATIC_LIBS=OFF - - make - - make install - ``` - - 2. Also you can using static libraries. - ``` - - mkdir build & cd build - - cmake ../ -DBoost_USE_STATIC_LIBS=ON -DROCKETMQ_USE_STATIC_LIBS=ON - - make - - make install - ``` -* Check verion - ``` - strings librocketmqclientpython.so |grep PYTHON_CLIENT_VERSION - ``` -#### macOS Mojave 10.14.2 -* Compile tools: - ``` - - make: 3.8 - - cmake 3.12 - - Apple LLVM(clang) 10 - ``` -* Install dependency: - - 1. python-devel - - 2. zlib-devel - - 3. boost-python - ``` - sh install_boostpython.sh - ``` - 4. [librocketmq](https://github.com/apache/rocketmq-client-cpp), choose one method below: - - - build from source: [Build and Install](https://github.com/apache/rocketmq-client-cpp/tree/master#build-and-install) - - - quick install: there are no cpp binary release for macos, below script can only be used for dev env. - ``` - mkdir rocketmqlib - cd rocketmqlib - wget https://opensource-rocketmq-client.oss-cn-hangzhou.aliyuncs.com/cpp-client/mac/1.2.4/librocketmq.tar.gz - tar -xzf librocketmq.tar.gz - cp librocketmq.dylib librocketmq.a /usr/local/lib/ - cp -r rocketmq /usr/local/include/ - ``` - - -* Make and install module manually - ``` - - mkdir build && cd build - - cmake ../ -DBoost_USE_STATIC_LIBS=OFF -DROCKETMQ_USE_STATIC_LIBS=OFF - - make - - make install - ``` -* Check verion - ``` - strings librocketmqclientpython.so |grep PYTHON_CLIENT_VERSION - ``` - ----------- -## How to use -- set LD_LIBRARY_PATH - ``` - export LD_LIBRARY_PATH=/usr/local/lib - ``` - -- import module - ``` - from librocketmqclientpython import * - ``` - -- create message by following interface: - ``` - - msg = CreateMessage("your_topic.") - - SetMessageBody(msg, "this_message_body.") - - SetMessageKeys(msg, "this_message_keys.") - - SetMessageTags(msg, "this_message_tag.") - ``` -- producer must invoke following interface: - ``` - - producer = CreateProducer("please_rename_unique_group_name"); - - SetProducerNameServerAddress(producer, "please_rename_unique_name_server") - - StartProducer(producer) - - SendMessageSync(producer, msg) - - ShutdownProducer(producer) - - DestroyProducer(producer) - ``` -- how to consumer messages - ``` - - def consumerMessage(msg, args): - - topic = GetMessageTopic(msg) - - body = GetMessageBody(msg) - - tags = GetMessageTags(msg) - - msgid = GetMessageId(msg) - - # handle message... - - return 0 - ``` -- pushconsumer must invoke following interface: - ``` - - consumer = CreatePushConsumer("please_rename_unique_group_name_1"); - - SetPushConsumerNameServerAddress(consumer, "please_rename_unique_name_server") - - Subscribe(consumer, "your_topic", "*") - - RegisterMessageCallback(consumer, consumerMessage, args) - - StartPushConsumer(consumer) - - ShutdownPushConsumer(consumer) - - DestroyPushConsumer(consumer) - ``` ----------- -## Demo -- sync producer - - python testProducer.py -- push consumer - - python testConsumer.py - diff --git a/doc/api-doc/consumer-push.md b/doc/api-doc/consumer-push.md deleted file mode 100644 index 6ee498e..0000000 --- a/doc/api-doc/consumer-push.md +++ /dev/null @@ -1,81 +0,0 @@ ----------- -## Api docs - -### 1. Push Consumer -* consumer = CreatePushConsumer(consumerGroup)
- - function description
- create a push consumer instance, by setting consumer group
- - - input
- consumerGroup: consumer group
- - - return
- consumer: consumer instance - -* SetPushConsumerNameServerAddress(consumer, namesrv)
- - function description
- set name srv address for the consumer instance
- - - input
- consumer: consumer intance
- namesrv: name srv address. like : 127.0.0.1:9876 - - - return : no
- -* Subscribe(consumer, topic, tag)
- - function description
- make consumer subscribe the topic and tag
- - - input
- consumer: consumer intance
- topic: topic name - tag: topic tag - -* RegisterMessageCallback(consumer, pyCallBack, pyArgs)
- - function description
- set callback for push consumer instance
- - - input
- consumer: consumer intance
- pyCallBack: py callback method. when message pulled, they would be send to a pyCallback method
- pyArgs: the arguments will be passed to pyCallBack - -* SetPushConsumerThreadCount(consumer, threadCount) - - function description
- set push consumer thread count
- - - input
- consumer: consumer intance
- threadCount: thread count - -* SetPushConsumerMessageBatchMaxSize(consumer, batchSize) - - function description
- set message count for one push
- - - input
- consumer: consumer intance
- batchSize: message count - -* SetPushConsumerInstanceName(consumer, instanceName) - - function description
- set consumer instance name
- - - input
- consumer: consumer intance
- instanceName: consumer instance name - -* SetPushConsumerSessionCredentials(consumer, accessKey, secretKey,channel) - - function description
- set consumer access keys
- - - input
- consumer: consumer intance
- accessKey: accessKey
- secretKey: secretKey
- channel: channel
- - - - - - diff --git a/doc/api-doc/message.md b/doc/api-doc/message.md deleted file mode 100644 index b727a6d..0000000 --- a/doc/api-doc/message.md +++ /dev/null @@ -1,135 +0,0 @@ ----------- -## Api docs - -### 1. Message -* message = CreateMessage("topicName")
- - function description
- create a message instance, by setting topic field
- - - input
- topicName: a topic name
- - - return
- a new message instance, after used it, you need call DestroyMessage(message)
- -* DestroyMessage(message)
- - function description
- destroy a message instance, delete memmory
- - - input
- message: message instance
- -* SetMessageTopic(message, topic)
- - function description
- set topic field value for the message
- - - input
- message: message instance
- topic: a topic name - -* SetMessageTags(message, tags)
- - function description
- set tag field value for the message
- - - input
- message: message instance
- tags: tag for the topic - -* SetMessageKeys(message, keys)
- - function description
- set key field value for the message
- - - input
- message: message instance
- keys: key for the topic - -* SetMessageBody(message, stringBody)
- - function description
- set body for the message
- - - input
- message: message instance
- body: message body as string - -* SetByteMessageBody(message, byteBody, byteLength)
- - function description
- set body for the message
- - - input
- message: message instance
- byteBody: message body as byte[] - byteLength: byteBody's length - -* SetMessageProperty(message, key, value)
- - function description
- set extend k-v for message
- - - input
- message: message instance
- key: string key - value: string value - -* SetMessageDelayTimeLevel(message, level)
- - function description
- set delay level
- - - input
- message: message instance
- level: delay level as int - - -### 2. MessageExt -* topic = GetMessageTopic(msgExt)
- - function description
- get topic name from a message instance
- - - input
- msgExt: message instance
- - return
- topic: topic name - -* tag = GetMessageTags(msgExt)
- - function description
- get tag from a message instance
- - - input
- msgExt: message instance
- - return
- tag: tag - -* key = GetMessageKeys(msgExt)
- - function description
- get message key from a message instance
- - - input
- msgExt: message instance
- - return
- key: message key - -* body = GetMessageBody(msgExt)
- - function description
- get message body from a message instance
- - - input
- msgExt: message instance
- - return
- body: message body as string - -* value = GetMessageProperty(msgExt, key)
- - function description
- get a message proprty value from a message instance
- - - input
- msgExt: message instance
- key: property key - - return
- value: property value as string - -* messageId = GetMessageId(msgExt)
- - function description
- get a message id from a message instance
- - - input
- msgExt: message instance
- - return
- messageId: message id as string \ No newline at end of file diff --git a/doc/api-doc/producer.md b/doc/api-doc/producer.md deleted file mode 100644 index 9d329f4..0000000 --- a/doc/api-doc/producer.md +++ /dev/null @@ -1,106 +0,0 @@ ----------- -## Api docs - -### Producer -* producer = CreateProducer("producerName")
- - function description
- create a producer instance
- - - input
- producerName: producer group name
- - - return
- a new producer instance, can send messages
- -* SetProducerNameServerAddress(producer, "namesrv address") - - function description
- set namesrv address for the producer instance
- - - input
- producer : a producer instance
- - namesrv address : like 127.0.0.1:9876
- - return : no
-* SetProducerInstanceName(producer, "instance name") - - function description
- set instance name for the producer - - - input
- producer : a producer instance
- intance name : a producer instance name
- - return : no
- -* SetProducerSessionCredentials(producer, accessKey, secretKey, channel) - - function description
- set access keys for accessing broker in the session - - - input
- producer : a producer instance
- accessKey : accessKey
- secretKey : secretKey
- channel : channel
- - return : no
- -* StartProducer(producer) - - function description
- start the producer instance - - - input
- producer : a producer instance
- - - return : no
- -* ShutdownProducer(producer) - - function description
- shutdown the producer instance - - - input
- producer : a producer instance
- - - return : no
- -* DestroyProducer(producer) - - function description
- destroy the producer instance - - - input
- producer : a producer instance
- - - return : no
- -* PySendResult result = SendMessageSync(producer, msg) - - function description
- send a message sync - - - input
- producer : a producer instance
- msg : a message instance
- - - return
- result.GetMsgId(): if send successfuly, it is the message id
- result.offset : message offset in broker
- result.sendStatus
- SEND_OK:
- SEND_FLUSH_DISK_TIMEOUT,
- SEND_FLUSH_SLAVE_TIMEOUT,
- SEND_SLAVE_NOT_AVAILABLE
- -* SendMessageOneway(producer, msg) - - function description
- send a message one way, no matter about the result - - - input
- producer : a producer instance
- msg : a message instance
- -* SendMessageOrderly(producer, msg, autoRetryTimes,arg, queueSelectorCallback) - - function description
- send a message orderly - - - input
- producer : a producer instance
- msg : a message instance
- autoRetryTimes: retry times when send fail
- arg: send args
- queueSelectorCallback: callback for which queue choose to send message to. return queue index start from 0 to (max queue count -1) - diff --git a/doc/quick-start.md b/doc/quick-start.md deleted file mode 100644 index 245eacb..0000000 --- a/doc/quick-start.md +++ /dev/null @@ -1,64 +0,0 @@ ----------- -## Qucik start - -* set cpp despendencies - ```bash - wget https://opensource-rocketmq-client.oss-cn-hangzhou.aliyuncs.com/cpp-client/linux/1.0.2/RHEL7.x/librocketmq.tar.gz - - tar -zxvf librocketmq.tar.gz - - cd librocketmq - - cp -R rocketmq /usr/local/include - - cd librocketmq.a librocketmq.so /usr/local/lib - - set LD_LIBRARY_PATH - - ``` - -* build python client from source. if you already had it, ignore this step - - [how to build](https://github.com/apache/rocketmq-client-python/blob/master/doc/Introduction.md) - - - copy the build result [librocketmqclientpython.so](#) to /usr/local/lib - -* how to produce a message
- ```python - from librocketmqclientpython import * - ### how to init a producer instance - def init_producer(): - producer = CreateProducer('your producer group name') - SetProducerNameServerAddress(producer, 'your name srv address') - StartProducer(producer) - return producer - ### how to send a message - def send(body): - msg = CreateMessage(topic) - SetMessageBody(msg, body) - result = SendMessageSync(producer, msg) - DestroyMessage(msg) - print 'done . msg id = ' + result.GetMsgId() - ``` - -* how to consume the message - ```python - from librocketmqclientpython import * - ## how to init a consumer intance - def build_consumer(_group, _topic, _tag): - consumer = CreatePushConsumer(_group) - SetPushConsumerNameServerAddress(consumer, name_srv) - SetPushConsumerThreadCount(consumer, 1) - Subscribe(consumer, _topic, _tag) - RegisterMessageCallback(consumer, callback) - StartPushConsumer(consumer) - print 'consumer is ready...' - return consumer - ## callback to consume the messages - def callback(msg): - print 'topic=%s' % GetMessageTopic(msg) - print 'tag=%s' % GetMessageTags(msg) - print 'body=%s' % GetMessageBody(msg) - print 'msg id=%s' % GetMessageId(msg) - print 'map.keys %s' % GetMessageKeys(msg) - return 0 - ``` \ No newline at end of file diff --git a/install_boostpython.sh b/install_boostpython.sh deleted file mode 100755 index 4ff9792..0000000 --- a/install_boostpython.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -VERSION=1.58.0 -BOOST=boost_1_58_0 - -if [ ! -d ${HOME}/${BOOST} ]; then - if [ -e ${HOME}/${BOOST}.tar.gz ]; then - echo "Find Packge ${HOME}/${BOOST}.tar.gz......." - else - wget -O ${HOME}/${BOOST}.tar.gz http://sourceforge.net/projects/boost/files/boost/${VERSION}/${BOOST}.tar.gz - fi - if [ $? -ne 0 ];then - exit 1 - fi - tar -xzf ${HOME}/${BOOST}.tar.gz -C ${HOME} - if [ $? -ne 0 ];then - exit 1 - fi -else - echo "Find Boost Source:${HOME}/${BOOST}, Build and install....." -fi - -cd ${HOME}/${BOOST} - -./bootstrap.sh --prefix=/usr/local --with-libraries=python -if [ $? -ne 0 ];then - exit 1 -fi -echo "Install boost static library...." -sudo ./b2 cflags="-fPIC" cxxflags="-fPIC -Wno-unused-local-typedefs -Wno-strict-aliasing" link=static \ - runtime-link=static --with-python \ - -a install -if [ $? -ne 0 ];then - exit 1 -fi -echo "Install boost dynamic library....." -sudo ./b2 cflags="-fPIC" cxxflags="-fPIC -Wno-unused-local-typedefs -Wno-strict-aliasing" link=shared \ - runtime-link=shared --with-python \ - -a install -if [ $? -ne 0 ];then - exit 1 -fi -echo "Finish build boost library." diff --git a/install_gtest.sh b/install_gtest.sh deleted file mode 100755 index 324e06f..0000000 --- a/install_gtest.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -VERSION=1.8.1 -PACKAGENAME=release-${VERSION} -GTEST=googletest-release-${VERSION} -release-1.8.1.tar.gz - -if [ ! -d ${HOME}/${GTEST} ]; then - if [ -e ${HOME}/${GTEST}.tar.gz ]; then - echo "Find Packge ${HOME}/${GTEST}.tar.gz......." - else - wget -O ${HOME}/${GTEST}.tar.gz https://github.com/abseil/googletest/archive/${PACKAGENAME}.tar.gz - fi - if [ $? -ne 0 ];then - exit 1 - fi - tar -xzf ${HOME}/${GTEST}.tar.gz -C ${HOME} - if [ $? -ne 0 ];then - exit 1 - fi -else - echo "Find GTest Source:${HOME}/${GTEST}, Build and install....." -fi - -cd ${HOME}/${GTEST} - -mkdir build; cd build -echo "Start build google test" - cmake .. -DCMAKE_CXX_FLAGS=-fPIC -DBUILD_STATIC_LIBS=ON -DBUILD_SHARED_LIBS=OFF - if [ $? -ne 0 ];then - exit 1 - fi - make - if [ $? -ne 0 ];then - exit 1 - fi - make install - - if [ ! -f /usr/local/lib/libgtest.a ] - then - echo "#######Error: Install gtest failed.#########" - exit 1 - fi - -echo "Finish build gtest library." diff --git a/manylinux.sh b/manylinux.sh new file mode 100755 index 0000000..0328103 --- /dev/null +++ b/manylinux.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cp /usr/local/lib/librocketmq.so /io/rocketmq/librocketmq.so + +# Build wheels +which linux32 && LINUX32=linux32 +$LINUX32 /opt/python/cp27-cp27mu/bin/python setup.py bdist_wheel + +# Audit wheels +for wheel in dist/*-linux_*.whl; do + auditwheel -v repair $wheel -w dist/ + rm $wheel +done diff --git a/package.sh b/package.sh deleted file mode 100755 index 404a4f4..0000000 --- a/package.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -function Help() -{ - echo "=========================================Package============================================" - echo "sh package.sh [shared]" - echo "usage: sh package.sh shared" - echo "shared: build python client by dynamic boost python and rocketmq library" - echo "default: build python client by static boost python and rocketmq library" - echo "=========================================Package============================================" - echo "" -} - -if [ $# -gt 0 ];then - if [ "$1" != "shared" ];then - #echo "unsupport para value $1, please see the help" - Help - exit 1 - fi -fi - -PACKAGE="rocketmq-client-python" -VERSION=$(cat src/PythonWrapper.h | grep PYTHON_CLIENT_VERSION | cut -f2 -d"\"") -CWD_DIR=$(cd "$(dirname "$0")"; pwd) -DEPLOY_BUILD_HOME=${CWD_DIR}/${PACKAGE} - -# ##==================================================================== -#make -rm -rf ${CWD_DIR}/tmpbuild -mkdir -p ${CWD_DIR}/tmpbuild -cd ${CWD_DIR}/tmpbuild -if [ "$1" = "shared" ]; then - echo "------------Build Client using dynamic library------------" - cmake ${CWD_DIR} -DBoost_USE_STATIC_LIBS=OFF -DROCKETMQ_USE_STATIC_LIBS=OFF - RMQ=$(cat CMakeCache.txt | grep ROCKETMQ_LIBRARIES:FILEPATH= | cut -f2 -d "=") - echo "Rocketmq Library:${RMQ}" -else - echo "-------------Build Client using static library-------------" - cmake ${CWD_DIR} -DBoost_USE_STATIC_LIBS=ON -DROCKETMQ_USE_STATIC_LIBS=ON -fi -if [ $? -ne 0 ];then - exit 1 -fi -make -if [ $? -ne 0 ];then - exit 1 -fi -cd ${CWD_DIR} -# ##==================================================================== -# # deploy -echo "Package Library...." -rm -rf ${DEPLOY_BUILD_HOME} -mkdir -p ${DEPLOY_BUILD_HOME}/lib -if [ "$1" = "shared" ];then - echo "Copy librocketmq to package...." - cp -rf ${RMQ} ${DEPLOY_BUILD_HOME}/lib/ - #cp -rf /usr/local/lib/libboost_python.*.so.* ${DEPLOY_BUILD_HOME}/lib/ -fi -cp -rf ${CWD_DIR}/bin/*.so ${DEPLOY_BUILD_HOME}/lib/ -cp -rf ${CWD_DIR}/sample ${DEPLOY_BUILD_HOME}/ -cp -rf ${CWD_DIR}/doc ${DEPLOY_BUILD_HOME}/ -cp -rf ${CWD_DIR}/changelog ${DEPLOY_BUILD_HOME}/ - - -cd ${CWD_DIR} && tar -cvzf ./${PACKAGE}-${VERSION}.tar.gz ./${PACKAGE} >/dev/null 2>&1 -rm -rf ${DEPLOY_BUILD_HOME} -# # ##==================================================================== -cd ${CWD_DIR}/tmpbuild -make clean -cd ${CWD_DIR} -rm -rf ${CWD_DIR}/tmpbuild diff --git a/project/CMakeLists.txt b/project/CMakeLists.txt deleted file mode 100644 index 152c5e3..0000000 --- a/project/CMakeLists.txt +++ /dev/null @@ -1,61 +0,0 @@ -#/* -#* Licensed to the Apache Software Foundation (ASF) under one or more -#* contributor license agreements. See the NOTICE file distributed with -#* this work for additional information regarding copyright ownership. -#* The ASF licenses this file to You under the Apache License, Version 2.0 -#* (the "License"); you may not use this file except in compliance with -#* the License. You may obtain a copy of the License at -#* -#* http://www.apache.org/licenses/LICENSE-2.0 -#* -#* Unless required by applicable law or agreed to in writing, software -#* distributed under the License is distributed on an "AS IS" BASIS, -#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#* See the License for the specific language governing permissions and -#* limitations under the License. -#*/ - -project(rocketmqclientpython) - -file(GLOB_RECURSE SRC_FILES ${CMAKE_SOURCE_DIR}/src/*) - -# subdirs -SET(SUB_DIRS) -file(GLOB children ${CMAKE_SOURCE_DIR}/src/*) -FOREACH (child ${children}) - IF (IS_DIRECTORY ${child}) - LIST(APPEND SUB_DIRS ${child}) - ENDIF () -ENDFOREACH () -LIST(APPEND SUB_DIRS ${CMAKE_SOURCE_DIR}/src) - -# include_directories -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${SUB_DIRS}) -#include_directories(/usr/local/include/rocketmq) -#include_directories(/usr/include/python2.7) -#include_directories(/System/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7) - - -# static -add_library(rocketmqclientpython_static STATIC ${SRC_FILES}) -set_target_properties(rocketmqclientpython_static PROPERTIES OUTPUT_NAME "rocketmqclientpython") -target_link_libraries(rocketmqclientpython_static ${deplibs}) -target_link_libraries(rocketmqclientpython_static ${ROCKETMQ_LIBRARIES}) -target_link_libraries(rocketmqclientpython_static ${Boost_LIBRARIES}) -target_link_libraries(rocketmqclientpython_static ${PYTHON_LIBRARIES}) -# shared -set(CMAKE_SHARED_LINKER_FLAGS "-fPIC -shared") -add_library(rocketmqclientpython_shared SHARED ${SRC_FILES}) -set_target_properties(rocketmqclientpython_shared PROPERTIES OUTPUT_NAME "rocketmqclientpython") -if (APPLE) - set_target_properties(rocketmqclientpython_shared PROPERTIES SUFFIX .so) -endif (APPLE) -target_link_libraries(rocketmqclientpython_shared ${deplibs}) -target_link_libraries(rocketmqclientpython_shared ${ROCKETMQ_LIBRARIES}) -target_link_libraries(rocketmqclientpython_shared ${Boost_LIBRARIES}) -target_link_libraries(rocketmqclientpython_shared ${PYTHON_LIBRARIES}) - -# install -install(TARGETS rocketmqclientpython_shared DESTINATION lib) - diff --git a/project/Makefile b/project/Makefile deleted file mode 100755 index c4be14f..0000000 --- a/project/Makefile +++ /dev/null @@ -1,98 +0,0 @@ -#/* -#* Licensed to the Apache Software Foundation (ASF) under one or more -#* contributor license agreements. See the NOTICE file distributed with -#* this work for additional information regarding copyright ownership. -#* The ASF licenses this file to You under the Apache License, Version 2.0 -#* (the "License"); you may not use this file except in compliance with -#* the License. You may obtain a copy of the License at -#* -#* http://www.apache.org/licenses/LICENSE-2.0 -#* -#* Unless required by applicable law or agreed to in writing, software -#* distributed under the License is distributed on an "AS IS" BASIS, -#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#* See the License for the specific language governing permissions and -#* limitations under the License. -#*/ - -##==================================================================== -# make release=0 debug版。 -# make release=1 release版。 -CXXFLAGS = -g -pthread -Wall -fPIC -Wno-deprecated -fno-strict-aliasing -fno-omit-frame-pointer -O0 -DNDEBUG - -ifeq ($(shell uname -m),x86_64) - CXXFLAGS += -m64 -else - CXXFLAGS += -m32 -endif - -##==================================================================== -PREFIX:=/usr/local -TOPDIR := .. -STATIC_TARGET := $(TOPDIR)/bin/librocketmqclientpython.a -SHARED_TARGET := $(TOPDIR)/bin/librocketmqclientpython.so -SRCDIR:=$(TOPDIR)/src -INCFILES:=$(wildcard $(TOPDIR)/include/*.h) -CPP_SRCDIR := $(SRCDIR) - -CPP_SRC := $(foreach dir,$(CPP_SRCDIR), $(wildcard $(dir)/*.cpp)) -STATIC_OBJ := $(filter-out %/dllmain.o,$(patsubst %.cpp, %.o, $(CPP_SRC))) -SHARED_OBJ := $(filter-out %/dllmain.lo,$(patsubst %.cpp, %.lo, $(CPP_SRC))) -VPATH:=$(CPP_SRCDIR) - -PYTHON_INCLUDE:=/usr/include/python2.7 -ROCKETMQ_INCLUDE:=/usr/local/include/rocketmq -PYTHON_LIB:=/usr/python/bin -PYTHON_LIBD:=/usr/python/lib -ROCKETMQ_LIBD:=/usr/local/lib -CPPFLAGS := -I$(TOPDIR)/include \ - $(addprefix -I,$(CPP_SRCDIR)) \ - $(addprefix -I,$(ROCKETMQ_INCLUDE)) \ - $(addprefix -I,$(PYTHON_INCLUDE)) - -LDFLAGS := -shared -Wl,-soname=librocketmqclientpython.so -pthread -fPIC -LIBPATH := $(addprefix -L,$(ROCKETMQ_LIBD) $(PYTHON_LIBD)) -LDLIBS := $(addprefix -l,rocketmq boost_python) - -CXX := g++ -AR := ar -ARFLAGS := rcs -LIBS:= $(foreach dir,$(ROCKETMQ_LIBD),$(wildcard $(dir)/*/lib/*.a)) -##==================================================================== -#include tool.mak -##==================================================================== -all:build-static build-shared - -build-static:$(STATIC_TARGET) - -build-shared:$(SHARED_TARGET) - -$(STATIC_TARGET):$(STATIC_OBJ) $(LIBS) - -$(SHARED_TARGET):$(SHARED_OBJ) - $(CXX) $(LDFLAGS) -o $@ $^ $(LIBPATH) $(LDLIBS) - -%.o:%.cpp - $(CXX) -c $(CXXFLAGS) $(CPPFLAGS) -o $@ $< - -%.lo:%.cpp - $(CXX) -c $(CXXFLAGS) $(CPPFLAGS) -o $@ $< - -rebuild:clean build - -test: - @echo $(LIBS) - -clean: - $(RM) -rf $(STATIC_OBJ) - $(RM) -rf $(SHARED_OBJ) - $(RM) -rf ipch *.sdf *.opensdf *.user *.suo - -install: $(STATIC_TARGET) $(SHARED_TARGET) - mkdir -p $(PREFIX)/lib - rm -rf $(PREFIX)/lib/librocketmqclientpython.a - rm -rf $(PREFIX)/lib/librocketmqclientpython.so - #cp -rf $(STATIC_TARGET) $(PREFIX)/lib/ - cp -rf $(SHARED_TARGET) $(PREFIX)/lib/ - @echo - @echo 'Install succeed, target directory is "'$(PREFIX)'".' diff --git a/project/tool.mak b/project/tool.mak deleted file mode 100644 index 6bbbb5c..0000000 --- a/project/tool.mak +++ /dev/null @@ -1,38 +0,0 @@ -#/* -#* Licensed to the Apache Software Foundation (ASF) under one or more -#* contributor license agreements. See the NOTICE file distributed with -#* this work for additional information regarding copyright ownership. -#* The ASF licenses this file to You under the Apache License, Version 2.0 -#* (the "License"); you may not use this file except in compliance with -#* the License. You may obtain a copy of the License at -#* -#* http://www.apache.org/licenses/LICENSE-2.0 -#* -#* Unless required by applicable law or agreed to in writing, software -#* distributed under the License is distributed on an "AS IS" BASIS, -#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#* See the License for the specific language governing permissions and -#* limitations under the License. -#*/ - -define BUILD_LIBRARY -$(if $(wildcard $@),@$(RM) $@) -$(if $(wildcard ar.mac),@$(RM) ar.mac) -$(if $(filter %.a, $^), -@echo CREATE $@ > ar.mac -@echo SAVE >> ar.mac -@echo END >> ar.mac -@$(AR) -M < ar.mac -) -$(if $(filter %.o,$^),@$(AR) -q $@ $(filter %.o, $^)) -$(if $(filter %.a, $^), -@echo OPEN $@ > ar.mac -$(foreach LIB, $(filter %.a, $^), -@echo ADDLIB $(LIB) >> ar.mac -) -@echo SAVE >> ar.mac -@echo END >> ar.mac -@$(AR) -M < ar.mac -@$(RM) ar.mac -) -endef diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..800dfe6 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +[pytest] +testpaths = tests +timeout = 600 diff --git a/rocketmq/__init__.py b/rocketmq/__init__.py new file mode 100644 index 0000000..2922526 --- /dev/null +++ b/rocketmq/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. \ No newline at end of file diff --git a/rocketmq/client.py b/rocketmq/client.py new file mode 100644 index 0000000..bb8e9d4 --- /dev/null +++ b/rocketmq/client.py @@ -0,0 +1,462 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import sys +import ctypes +from enum import IntEnum +from collections import namedtuple + +from .ffi import ( + dll, _CSendResult, MSG_CALLBACK_FUNC, MessageModel, TRANSACTION_CHECK_CALLBACK, + LOCAL_TRANSACTION_EXECUTE_CALLBACK, TraceModel +) +from .exceptions import ( + ffi_check, NullPointerException, +) +from .consts import MessageProperty + +__all__ = ['SendStatus', 'Message', 'ReceivedMessage', 'Producer', 'PushConsumer', 'TransactionMQProducer', + 'TransactionStatus', 'ConsumeStatus'] + +PY2 = sys.version_info[0] == 2 +if PY2: + text_type = unicode + binary_type = str +else: + text_type = str + binary_type = bytes + +SendResult = namedtuple('SendResult', ['status', 'msg_id', 'offset']) + + +class SendStatus(IntEnum): + OK = 0 + FLUSH_DISK_TIMEOUT = 1 + FLUSH_SLAVE_TIMEOUT = 2 + SLAVE_NOT_AVAILABLE = 3 + + +class TransactionStatus(IntEnum): + COMMIT = 0 + ROLLBACK = 1 + UNKNOWN = 2 + + +class ConsumeStatus(IntEnum): + CONSUME_SUCCESS = 0 + RECONSUME_LATER = 1 + + +def _to_bytes(s): + if isinstance(s, text_type): + return s.encode('utf-8') + return s + + +class Message(object): + def __init__(self, topic): + self._handle = dll.CreateMessage(_to_bytes(topic)) + + def __del__(self): + dll.DestroyMessage(self._handle) + + def set_keys(self, keys): + ffi_check(dll.SetMessageKeys(self._handle, _to_bytes(keys))) + + def set_tags(self, tags): + ffi_check(dll.SetMessageTags(self._handle, _to_bytes(tags))) + + def set_body(self, body): + ffi_check(dll.SetMessageBody(self._handle, _to_bytes(body))) + + def set_property(self, key, value): + ffi_check(dll.SetMessageProperty(self._handle, _to_bytes(key), _to_bytes(value))) + + def set_delay_time_level(self, delay_time_level): + ffi_check(dll.SetDelayTimeLevel(self._handle, delay_time_level)) + + @property + def _as_parameter_(self): + return self._handle + + +def maybe_decode(val): + if isinstance(val, binary_type): + return val.decode('utf-8') + elif isinstance(val, text_type): + return val + raise TypeError('Expects string types, but got %s', type(val)) + + +class ReceivedMessage(object): + def __init__(self, handle): + self._handle = handle + + @property + def topic(self): + return maybe_decode(dll.GetMessageTopic(self._handle)) + + @property + def tags(self): + return dll.GetMessageTags(self._handle) + + @property + def keys(self): + return dll.GetMessageKeys(self._handle) + + @property + def body(self): + return dll.GetMessageBody(self._handle) + + @property + def id(self): + return maybe_decode(dll.GetMessageId(self._handle)) + + @property + def delay_time_level(self): + return dll.GetMessageDelayTimeLevel(self._handle) + + @property + def queue_id(self): + return dll.GetMessageQueueId(self._handle) + + @property + def reconsume_times(self): + return dll.GetMessageReconsumeTimes(self._handle) + + @property + def store_size(self): + return dll.GetMessageStoreSize(self._handle) + + @property + def born_timestamp(self): + return dll.GetMessageBornTimestamp(self._handle) + + @property + def store_timestamp(self): + return dll.GetMessageStoreTimestamp(self._handle) + + @property + def queue_offset(self): + return dll.GetMessageQueueOffset(self._handle) + + @property + def commit_log_offset(self): + return dll.GetMessageCommitLogOffset(self._handle) + + @property + def prepared_transaction_offset(self): + return dll.GetMessagePreparedTransactionOffset(self._handle) + + def get_property(self, prop): + if isinstance(prop, MessageProperty): + prop = prop.value + val = dll.GetMessageProperty(self._handle, _to_bytes(prop)) + return val + + def __getitem__(self, key): + return self.get_property(key) + + def __str__(self): + return self.body.decode('utf-8') + + def __bytes__(self): + return self.body + + def __repr__(self): + return ''.format( + repr(self.topic), + repr(self.id), + repr(self.body), + ) + + +class Producer(object): + def __init__(self, group_id, orderly=False, timeout=None, compress_level=None, max_message_size=None): + if orderly: + self._handle = dll.CreateOrderlyProducer(_to_bytes(group_id)) + else: + self._handle = dll.CreateProducer(_to_bytes(group_id)) + if self._handle is None: + raise NullPointerException('Returned null pointer when create Producer') + if timeout is not None: + self.set_timeout(timeout) + if compress_level is not None: + self.set_compress_level(compress_level) + if max_message_size is not None: + self.set_max_message_size(max_message_size) + self._callback_refs = [] + + def __enter__(self): + self.start() + + def __exit__(self, exec_type, value, traceback): + self.shutdown() + + def send_sync(self, msg): + c_result = _CSendResult() + ffi_check(dll.SendMessageSync(self._handle, msg, ctypes.pointer(c_result))) + return SendResult( + SendStatus(c_result.sendStatus), + c_result.msgId.decode('utf-8'), + c_result.offset + ) + + def send_oneway(self, msg): + ffi_check(dll.SendMessageOneway(self._handle, msg)) + + def send_orderly_with_sharding_key(self, msg, sharding_key): + c_result = _CSendResult() + ffi_check( + dll.SendMessageOrderlyByShardingKey(self._handle, msg, _to_bytes(sharding_key), ctypes.pointer(c_result))) + return SendResult( + SendStatus(c_result.sendStatus), + c_result.msgId.decode('utf-8'), + c_result.offset + ) + + def set_group(self, group_name): + ffi_check(dll.SetProducerGroupName(self._handle, _to_bytes(group_name))) + + def set_instance_name(self, name): + ffi_check(dll.SetProducerInstanceName(self._handle, _to_bytes(name))) + + def set_name_server_address(self, addr): + ffi_check(dll.SetProducerNameServerAddress(self._handle, _to_bytes(addr))) + + def set_name_server_domain(self, domain): + ffi_check(dll.SetProducerNameServerDomain(self._handle, _to_bytes(domain))) + + def set_session_credentials(self, access_key, access_secret, channel): + ffi_check(dll.SetProducerSessionCredentials( + self._handle, + _to_bytes(access_key), + _to_bytes(access_secret), + _to_bytes(channel) + )) + + def set_timeout(self, timeout): + ffi_check(dll.SetProducerSendMsgTimeout(self._handle, timeout)) + + def set_compress_level(self, level): + ffi_check(dll.SetProducerCompressLevel(self._handle, level)) + + def set_max_message_size(self, max_size): + ffi_check(dll.SetProducerMaxMessageSize(self._handle, max_size)) + + def set_message_trace(self, message_trace): + ffi_check(dll.SetProducerMessageTrace(self._handle, message_trace and TraceModel.OPEN or TraceModel.CLOSE)) + + def set_ssl_enable(self, enable): + ssl_enable_code = 1 if enable else 0 + ffi_check(dll.SetProducerSsl(self._handle, ssl_enable_code)) + + def set_ssl_property_file(self, file_path): + ffi_check(dll.SetProducerSslPropertyFile(self._handle, _to_bytes(file_path))) + + def start(self): + ffi_check(dll.StartProducer(self._handle)) + + def shutdown(self): + ffi_check(dll.ShutdownProducer(self._handle)) + + +class TransactionMQProducer(Producer): + def __init__(self, group_id, checker_callback, user_args=None, timeout=None, compress_level=None, + max_message_size=None): + super(TransactionMQProducer, self).__init__(group_id, timeout, compress_level, max_message_size) + self._callback_refs = [] + + def _on_check(producer, c_message, user_data): + exc = None + try: + py_message = ReceivedMessage(c_message) + check_result = checker_callback(py_message) + if check_result != TransactionStatus.UNKNOWN and check_result != TransactionStatus.COMMIT \ + and check_result != TransactionStatus.ROLLBACK: + raise ValueError( + 'Check transaction status error, please use enum \'TransactionStatus\' as response') + return check_result + except BaseException as e: + exc = e + return TransactionStatus.UNKNOWN + finally: + if exc: + raise exc + + transaction_checker_callback = TRANSACTION_CHECK_CALLBACK(_on_check) + self._callback_refs.append(transaction_checker_callback) + + self._handle = dll.CreateTransactionProducer(_to_bytes(group_id), transaction_checker_callback, user_args) + if self._handle is None: + raise NullPointerException('Returned null pointer when create transaction producer') + if timeout is not None: + self.set_timeout(timeout) + if compress_level is not None: + self.set_compress_level(compress_level) + if max_message_size is not None: + self.set_max_message_size(max_message_size) + + def __enter__(self): + self.start() + + def __exit__(self, exec_type, value, traceback): + self.shutdown() + + def set_name_server_address(self, addr): + ffi_check(dll.SetProducerNameServerAddress(self._handle, _to_bytes(addr))) + + def set_message_trace(self, message_trace): + ffi_check(dll.SetProducerMessageTrace(self._handle, message_trace and TraceModel.OPEN or TraceModel.CLOSE)) + + def start(self): + ffi_check(dll.StartProducer(self._handle)) + + def send_message_in_transaction(self, message, local_execute, user_args=None): + + def _on_local_execute(producer, c_message, usr_args): + exc = None + try: + py_message = ReceivedMessage(c_message) + local_result = local_execute(py_message, usr_args) + if local_result != TransactionStatus.UNKNOWN and local_result != TransactionStatus.COMMIT \ + and local_result != TransactionStatus.ROLLBACK: + raise ValueError( + 'Local transaction status error, please use enum \'TransactionStatus\' as response') + return local_result + except BaseException as e: + exc = e + return TransactionStatus.UNKNOWN + finally: + if exc: + raise exc + + local_execute_callback = LOCAL_TRANSACTION_EXECUTE_CALLBACK(_on_local_execute) + self._callback_refs.append(local_execute_callback) + + result = _CSendResult() + try: + ffi_check( + dll.SendMessageTransaction(self._handle, + message, + local_execute_callback, + user_args, + ctypes.pointer(result))) + finally: + self._callback_refs.remove(local_execute_callback) + + return SendResult( + SendStatus(result.sendStatus), + result.msgId.decode('utf-8'), + result.offset + ) + + +class PushConsumer(object): + def __init__(self, group_id, orderly=False, message_model=MessageModel.CLUSTERING): + self._handle = dll.CreatePushConsumer(_to_bytes(group_id)) + if self._handle is None: + raise NullPointerException('Returned null pointer when create PushConsumer') + self._orderly = orderly + self.set_message_model(message_model) + self._callback_refs = [] + + def __enter__(self): + self.start() + + def __exit__(self, exec_type, value, traceback): + self.shutdown() + + def set_message_model(self, model): + ffi_check(dll.SetPushConsumerMessageModel(self._handle, model)) + + def start(self): + ffi_check(dll.StartPushConsumer(self._handle)) + + def shutdown(self): + ffi_check(dll.ShutdownPushConsumer(self._handle)) + + def set_group(self, group_id): + ffi_check(dll.SetPushConsumerGroupID(self._handle, _to_bytes(group_id))) + + def set_name_server_address(self, addr): + ffi_check(dll.SetPushConsumerNameServerAddress(self._handle, _to_bytes(addr))) + + def set_name_server_domain(self, domain): + ffi_check(dll.SetPushConsumerNameServerDomain(self._handle, _to_bytes(domain))) + + def set_session_credentials(self, access_key, access_secret, channel): + ffi_check(dll.SetPushConsumerSessionCredentials( + self._handle, + _to_bytes(access_key), + _to_bytes(access_secret), + _to_bytes(channel) + )) + + def set_ssl_enable(self, enable): + ssl_enable_code = 1 if enable else 0 + ffi_check(dll.SetPushConsumerSsl(self._handle, ssl_enable_code)) + + def set_ssl_property_file(self, file_path): + ffi_check(dll.SetPushConsumerSslPropertyFile(self._handle, _to_bytes(file_path))) + + def subscribe(self, topic, callback, expression='*'): + def _on_message(consumer, msg): + exc = None + try: + consume_result = callback(ReceivedMessage(msg)) + if consume_result != ConsumeStatus.CONSUME_SUCCESS and consume_result != ConsumeStatus.RECONSUME_LATER: + raise ValueError('Consume status error, please use enum \'ConsumeStatus\' as response') + return consume_result + except BaseException as e: + exc = e + return ConsumeStatus.RECONSUME_LATER + finally: + if exc: + raise exc + + ffi_check(dll.Subscribe(self._handle, _to_bytes(topic), _to_bytes(expression))) + self._register_callback(_on_message) + + def _register_callback(self, callback): + if self._orderly: + register_func = dll.RegisterMessageCallbackOrderly + else: + register_func = dll.RegisterMessageCallback + + func = MSG_CALLBACK_FUNC(callback) + self._callback_refs.append(func) + ffi_check(register_func(self._handle, func)) + + def _unregister_callback(self): + if self._orderly: + ffi_check(dll.UnregisterMessageCallbackOrderly(self._handle)) + ffi_check(dll.UnregisterMessageCallback(self._handle)) + self._callback_refs = [] + + def set_thread_count(self, thread_count): + ffi_check(dll.SetPushConsumerThreadCount(self._handle, thread_count)) + + def set_message_batch_max_size(self, max_size): + ffi_check(dll.SetPushConsumerMessageBatchMaxSize(self._handle, max_size)) + + def set_instance_name(self, name): + ffi_check(dll.SetPushConsumerInstanceName(self._handle, _to_bytes(name))) + + def set_message_trace(self, message_trace): + ffi_check(dll.SetPushConsumerMessageTrace(self._handle, message_trace and TraceModel.OPEN or TraceModel.CLOSE)) diff --git a/rocketmq/consts.py b/rocketmq/consts.py new file mode 100644 index 0000000..41b1b66 --- /dev/null +++ b/rocketmq/consts.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from enum import Enum + + +class MessageProperty(Enum): + TRACE_SWITCH = "TRACE_ON" + MSG_REGION = "MSG_REGION" + KEYS = "KEYS" + TAGS = "TAGS" + WAIT_STORE_MSG_OK = "WAIT" + DELAY_TIME_LEVEL = "DELAY" + RETRY_TOPIC = "RETRY_TOPIC" + REAL_TOPIC = "REAL_TOPIC" + REAL_QUEUE_ID = "REAL_QID" + TRANSACTION_PREPARED = "TRAN_MSG" + PRODUCER_GROUP = "PGROUP" + MIN_OFFSET = "MIN_OFFSET" + MAX_OFFSET = "MAX_OFFSET" + BUYER_ID = "BUYER_ID" + ORIGIN_MESSAGE_ID = "ORIGIN_MESSAGE_ID" + TRANSFER_FLAG = "TRANSFER_FLAG" + CORRECTION_FLAG = "CORRECTION_FLAG" + MQ2_FLAG = "MQ2_FLAG" + RECONSUME_TIME = "RECONSUME_TIME" + UNIQ_CLIENT_MESSAGE_ID_KEYIDX = "UNIQ_KEY" + MAX_RECONSUME_TIMES = "MAX_RECONSUME_TIMES" + CONSUME_START_TIMESTAMP = "CONSUME_START_TIME" diff --git a/rocketmq/exceptions.py b/rocketmq/exceptions.py new file mode 100644 index 0000000..293d60b --- /dev/null +++ b/rocketmq/exceptions.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import re + +from .ffi import dll, _CStatus + + +_EXCEPTION_MAP = {} + + +def _register(status_code): + def register(cls): + _EXCEPTION_MAP[status_code] = cls + return cls + return register + + +def ffi_check(status_code): + if status_code == _CStatus.OK: + return + exc_cls = _EXCEPTION_MAP.get(status_code, RocketMQException) + msg = dll.GetLatestErrorMessage() + if msg is not None: + msg = msg.decode('utf-8') + msg = re.sub('<.*?(rocketmq-client-cpp/)(.*)>', '\\1\\2', msg) + if msg.startswith('msg: '): + msg = msg[5:] + raise exc_cls(msg) + + +class RocketMQException(Exception): + '''RocketMQ exception base class''' + pass + + +@_register(_CStatus.NULL_POINTER) +class NullPointerException(RocketMQException): + pass + + +@_register(_CStatus.MALLOC_FAILED) +class MallocFailed(RocketMQException): + pass + + +class ProducerException(RocketMQException): + pass + + +@_register(_CStatus.PRODUCER_START_FAILED) +class ProducerStartFailed(ProducerException): + pass + + +@_register(_CStatus.PRODUCER_SEND_SYNC_FAILED) +class ProducerSendSyncFailed(ProducerException): + pass + + +@_register(_CStatus.PRODUCER_SEND_ONEWAY_FAILED) +class ProducerSendOnewayFailed(ProducerException): + pass + + +@_register(_CStatus.PRODUCER_SEND_ORDERLY_FAILED) +class ProducerSendOrderlyFailed(ProducerException): + pass + + +class ProducerSendAsyncFailed(ProducerException): + def __init__(self, msg, error, file, line, type): + super(ProducerSendAsyncFailed, self).__init__(msg) + self.error = error + self.file = file + self.line = line + self.type = type + + +class ConsumerException(RocketMQException): + pass + + +@_register(_CStatus.PUSH_CONSUMER_START_FAILED) +class PushConsumerStartFailed(ConsumerException): + pass + diff --git a/rocketmq/ffi.py b/rocketmq/ffi.py new file mode 100644 index 0000000..0eca2ca --- /dev/null +++ b/rocketmq/ffi.py @@ -0,0 +1,291 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import os +import sys +import ctypes +from ctypes.util import find_library +from ctypes import c_char, c_char_p, c_void_p, c_int, c_long, c_longlong, Structure, POINTER +from enum import IntEnum + +_DYLIB_SUFFIX = '.so' +if sys.platform.lower() == 'darwin': + _DYLIB_SUFFIX = '.dylib' +elif sys.platform.lower() == 'win32': + raise NotImplementedError('rocketmq-python does not support Windows') + +_CURR_DIR = os.path.abspath(os.path.dirname(__file__)) +_PKG_DYLIB_PATH = os.path.join(_CURR_DIR, 'librocketmq' + _DYLIB_SUFFIX) +_DYLIB_PATH = find_library('rocketmq') +if os.path.exists(_PKG_DYLIB_PATH): + # Prefer packaged librocketmq dylib + _DYLIB_PATH = _PKG_DYLIB_PATH + +if not _DYLIB_PATH: + raise ImportError('rocketmq dynamic library not found') + +dll = ctypes.cdll.LoadLibrary(_DYLIB_PATH) + + +class CtypesEnum(IntEnum): + """A ctypes-compatible IntEnum superclass.""" + + @classmethod + def from_param(cls, obj): + return int(obj) + + +class _CStatus(CtypesEnum): + OK = 0 + NULL_POINTER = 1 + MALLOC_FAILED = 2 + # producer + PRODUCER_START_FAILED = 10 + PRODUCER_SEND_SYNC_FAILED = 11 + PRODUCER_SEND_ONEWAY_FAILED = 12 + PRODUCER_SEND_ORDERLY_FAILED = 13 + PRODUCER_SEND_ASYNC_FAILED = 14 + # push consumer + PUSH_CONSUMER_START_FAILED = 20 + + NOT_SUPPORT_NOW = -1 + + +class _CLogLevel(CtypesEnum): + FATAL = 1 + ERROR = 2 + WARN = 3 + INFO = 4 + DEBUG = 5 + TRACE = 6 + LEVEL_NUM = 7 + + +class MessageModel(CtypesEnum): + BROADCASTING = 0 + CLUSTERING = 1 + + +class TraceModel(CtypesEnum): + OPEN = 0 + CLOSE = 1 + + +class _CSendResult(Structure): + _fields_ = [ + ('sendStatus', c_int), + ('msgId', c_char * 256), + ('offset', c_longlong), + ] + + +class _CMessageQueue(Structure): + _fields_ = [ + ('topic', c_char * 512), + ('brokerName', c_char * 256), + ('queueId', c_int), + ] + + +class _CMQException(Structure): + _fields_ = [ + ('error', c_int), + ('line', c_int), + ('file', c_char * 512), + ('msg', c_char * 512), + ('type', c_char * 512), + ] + + +# Message +dll.CreateMessage.argtypes = [c_char_p] +dll.CreateMessage.restype = c_void_p +dll.DestroyMessage.argtypes = [c_void_p] +dll.DestroyMessage.restype = _CStatus +dll.SetMessageKeys.argtypes = [c_void_p, c_char_p] +dll.SetMessageKeys.restype = _CStatus +dll.SetMessageTags.argtypes = [c_void_p, c_char_p] +dll.SetMessageTags.restype = _CStatus +dll.SetMessageBody.argtypes = [c_void_p, c_char_p] +dll.SetMessageBody.restype = _CStatus +dll.SetByteMessageBody.argtypes = [c_void_p, c_char_p, c_int] +dll.SetByteMessageBody.restype = _CStatus +dll.SetMessageProperty.argtypes = [c_void_p, c_char_p, c_char_p] +dll.SetMessageProperty.restype = _CStatus +dll.SetDelayTimeLevel.argtypes = [c_void_p, c_int] +dll.SetDelayTimeLevel.restype = _CStatus +dll.GetMessageTopic.argtypes = [c_void_p] +dll.GetMessageTopic.restype = c_char_p +dll.GetMessageTags.argtypes = [c_void_p] +dll.GetMessageTags.restype = c_char_p +dll.GetMessageKeys.argtypes = [c_void_p] +dll.GetMessageKeys.restype = c_char_p +dll.GetMessageBody.argtypes = [c_void_p] +dll.GetMessageBody.restype = c_char_p +dll.GetMessageProperty.argtypes = [c_void_p, c_char_p] +dll.GetMessageProperty.restype = c_char_p +dll.GetMessageId.argtypes = [c_void_p] +dll.GetMessageId.restype = c_char_p +dll.GetMessageDelayTimeLevel.argtypes = [c_void_p] +dll.GetMessageDelayTimeLevel.restype = c_int +dll.GetMessageQueueId.argtypes = [c_void_p] +dll.GetMessageQueueId.restype = c_int +dll.GetMessageReconsumeTimes.argtypes = [c_void_p] +dll.GetMessageReconsumeTimes.restype = c_int +dll.GetMessageStoreSize.argtypes = [c_void_p] +dll.GetMessageStoreSize.restype = c_int +dll.GetMessageBornTimestamp.argtypes = [c_void_p] +dll.GetMessageBornTimestamp.restype = c_longlong +dll.GetMessageStoreTimestamp.argtypes = [c_void_p] +dll.GetMessageStoreTimestamp.restype = c_longlong +dll.GetMessageQueueOffset.argtypes = [c_void_p] +dll.GetMessageQueueOffset.restype = c_longlong +dll.GetMessageCommitLogOffset.argtypes = [c_void_p] +dll.GetMessageCommitLogOffset.restype = c_longlong +dll.GetMessagePreparedTransactionOffset.argtypes = [c_void_p] +dll.GetMessagePreparedTransactionOffset.restype = c_longlong +dll.CreateBatchMessage.argtypes = [] +dll.CreateBatchMessage.restype = c_void_p +dll.AddMessage.argtypes = [c_void_p, c_void_p] +dll.AddMessage.restype = _CStatus +dll.DestroyBatchMessage.argtypes = [c_void_p] +dll.DestroyBatchMessage.restype = _CStatus + +# Producer +QUEUE_SELECTOR_CALLBACK = ctypes.CFUNCTYPE(c_int, c_int, c_void_p, c_void_p) +SEND_SUCCESS_CALLBACK = ctypes.CFUNCTYPE(None, POINTER(_CSendResult)) +SEND_EXCEPTION_CALLBACK = ctypes.CFUNCTYPE(None, _CMQException) +TRANSACTION_CHECK_CALLBACK = ctypes.CFUNCTYPE(c_int, c_void_p, c_void_p, c_void_p) +LOCAL_TRANSACTION_EXECUTE_CALLBACK = ctypes.CFUNCTYPE(c_int, c_void_p, c_void_p, c_void_p) + +dll.CreateProducer.argtypes = [c_char_p] +dll.CreateProducer.restype = c_void_p +dll.CreateOrderlyProducer.argtypes = [c_char_p] +dll.CreateOrderlyProducer.restype = c_void_p +dll.DestroyProducer.argtypes = [c_void_p] +dll.DestroyProducer.restype = _CStatus +dll.StartProducer.argtypes = [c_void_p] +dll.StartProducer.restype = _CStatus +dll.ShutdownProducer.argtypes = [c_void_p] +dll.ShutdownProducer.restype = _CStatus +dll.SetProducerNameServerAddress.argtypes = [c_void_p, c_char_p] +dll.SetProducerNameServerAddress.restype = _CStatus +dll.SetProducerNameServerDomain.argtypes = [c_void_p, c_char_p] +dll.SetProducerNameServerDomain.restype = _CStatus +dll.SetProducerGroupName.argtypes = [c_void_p, c_char_p] +dll.SetProducerGroupName.restype = _CStatus +dll.SetProducerInstanceName.argtypes = [c_void_p, c_char_p] +dll.SetProducerInstanceName.restype = _CStatus +dll.SetProducerSessionCredentials.argtypes = [c_void_p, c_char_p, c_char_p, c_char_p] +dll.SetProducerSessionCredentials.restype = _CStatus +dll.SetProducerLogPath.argtypes = [c_void_p, c_char_p] +dll.SetProducerLogPath.restype = _CStatus +dll.SetProducerLogFileNumAndSize.argtypes = [c_void_p, c_int, c_long] +dll.SetProducerLogFileNumAndSize.restype = _CStatus +dll.SetProducerLogLevel.argtypes = [c_void_p, _CLogLevel] +dll.SetProducerLogLevel.restype = _CStatus +dll.SetProducerSendMsgTimeout.argtypes = [c_void_p, c_int] +dll.SetProducerSendMsgTimeout.restype = _CStatus +dll.SetProducerCompressLevel.argtypes = [c_void_p, c_int] +dll.SetProducerCompressLevel.restype = _CStatus +dll.SetProducerMaxMessageSize.argtypes = [c_void_p, c_int] +dll.SetProducerMaxMessageSize.restype = _CStatus +dll.SetProducerMessageTrace.argtypes = [c_void_p, TraceModel] +dll.SetProducerMessageTrace.restype = _CStatus +try: + dll.SetProducerSsl.argtypes = [c_void_p, c_int] + dll.SetProducerSsl.restype = _CStatus + dll.SetProducerSslPropertyFile.argtypes = [c_void_p, c_char_p] + dll.SetProducerSslPropertyFile.restype = _CStatus +except AttributeError: + pass +dll.SendMessageSync.argtypes = [c_void_p, c_void_p, POINTER(_CSendResult)] +dll.SendMessageSync.restype = _CStatus +dll.SendMessageOneway.argtypes = [c_void_p, c_void_p] +dll.SendMessageOneway.restype = _CStatus +dll.SendBatchMessage.argtypes = [c_void_p, c_void_p, POINTER(_CSendResult)] +dll.SendBatchMessage.restype = _CStatus + +dll.SendMessageOrderlyByShardingKey.argtypes = [c_void_p, c_void_p, c_char_p, POINTER(_CSendResult)] +dll.SendMessageOrderlyByShardingKey.restype = _CStatus + +dll.CreateTransactionProducer.argtypes = [c_char_p, TRANSACTION_CHECK_CALLBACK, c_void_p] +dll.CreateTransactionProducer.restype = c_void_p + +dll.SendMessageTransaction.argtypes = [c_void_p, c_void_p, LOCAL_TRANSACTION_EXECUTE_CALLBACK, c_void_p, + POINTER(_CSendResult)] +dll.SendMessageTransaction.restype = c_int + +# Push Consumer +MSG_CALLBACK_FUNC = ctypes.CFUNCTYPE(c_int, c_void_p, c_void_p) +dll.CreatePushConsumer.argtypes = [c_char_p] +dll.CreatePushConsumer.restype = c_void_p +dll.DestroyPushConsumer.argtypes = [c_void_p] +dll.DestroyPushConsumer.restype = _CStatus +dll.StartPushConsumer.argtypes = [c_void_p] +dll.StartPushConsumer.restype = _CStatus +dll.ShutdownPushConsumer.argtypes = [c_void_p] +dll.ShutdownPushConsumer.restype = _CStatus +dll.SetPushConsumerGroupID.argtypes = [c_void_p, c_char_p] +dll.SetPushConsumerGroupID.restype = _CStatus +dll.GetPushConsumerGroupID.argtypes = [c_void_p] +dll.GetPushConsumerGroupID.restype = c_char_p +dll.SetPushConsumerNameServerAddress.argtypes = [c_void_p, c_char_p] +dll.SetPushConsumerNameServerAddress.restype = _CStatus +dll.SetPushConsumerNameServerDomain.argtypes = [c_void_p, c_char_p] +dll.SetPushConsumerNameServerDomain.restype = _CStatus +dll.Subscribe.argtypes = [c_void_p, c_char_p, c_char_p] +dll.Subscribe.restype = _CStatus +dll.RegisterMessageCallbackOrderly.argtypes = [c_void_p, MSG_CALLBACK_FUNC] +dll.RegisterMessageCallbackOrderly.restype = _CStatus +dll.RegisterMessageCallback.argtypes = [c_void_p, MSG_CALLBACK_FUNC] +dll.RegisterMessageCallback.restype = _CStatus +dll.UnregisterMessageCallbackOrderly.argtypes = [c_void_p] +dll.UnregisterMessageCallbackOrderly.restype = _CStatus +dll.UnregisterMessageCallback.argtypes = [c_void_p] +dll.UnregisterMessageCallback.restype = _CStatus +dll.SetPushConsumerThreadCount.argtypes = [c_void_p, c_int] +dll.SetPushConsumerThreadCount.restype = _CStatus +dll.SetPushConsumerMessageBatchMaxSize.argtypes = [c_void_p, c_int] +dll.SetPushConsumerMessageBatchMaxSize.restype = _CStatus +dll.SetPushConsumerInstanceName.argtypes = [c_void_p, c_char_p] +dll.SetPushConsumerInstanceName.restype = _CStatus +dll.SetPushConsumerSessionCredentials.argtypes = [c_void_p, c_char_p, c_char_p, c_char_p] +dll.SetPushConsumerSessionCredentials.restype = _CStatus +dll.SetPushConsumerLogPath.argtypes = [c_void_p, c_char_p] +dll.SetPushConsumerLogPath.restype = _CStatus +dll.SetPushConsumerLogFileNumAndSize.argtypes = [c_void_p, c_int, c_long] +dll.SetPushConsumerLogFileNumAndSize.restype = _CStatus +dll.SetPushConsumerLogLevel.argtypes = [c_void_p, _CLogLevel] +dll.SetPushConsumerLogLevel.restype = _CStatus +dll.SetPushConsumerMessageModel.argtypes = [c_void_p, MessageModel] +dll.SetPushConsumerMessageModel.restype = _CStatus +dll.SetPushConsumerLogLevel.restype = _CStatus +dll.SetPushConsumerMessageTrace.argtypes = [c_void_p, TraceModel] +dll.SetPushConsumerMessageTrace.restype = _CStatus +try: + dll.SetPushConsumerSsl.argtypes = [c_void_p, c_int] + dll.SetPushConsumerSsl.restype = _CStatus + dll.SetPushConsumerSslPropertyFile.argtypes = [c_void_p, c_char_p] + dll.SetPushConsumerSslPropertyFile.restype = _CStatus +except AttributeError: + pass + +# Misc +dll.GetLatestErrorMessage.argtypes = [] +dll.GetLatestErrorMessage.restype = c_char_p diff --git a/sample/__init__.py b/sample/__init__.py deleted file mode 100644 index 6cf9b1e..0000000 --- a/sample/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -#/* -#* Licensed to the Apache Software Foundation (ASF) under one or more -#* contributor license agreements. See the NOTICE file distributed with -#* this work for additional information regarding copyright ownership. -#* The ASF licenses this file to You under the Apache License, Version 2.0 -#* (the "License"); you may not use this file except in compliance with -#* the License. You may obtain a copy of the License at -#* -#* http://www.apache.org/licenses/LICENSE-2.0 -#* -#* Unless required by applicable law or agreed to in writing, software -#* distributed under the License is distributed on an "AS IS" BASIS, -#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#* See the License for the specific language governing permissions and -#* limitations under the License. -#*/ - -import sys -sys.path.append('/usr/local/lib') -print("__________Python Version:___________") -print(sys.version) -print("______Add Path /usr/local/lib_______") diff --git a/sample/base.py b/sample/base.py deleted file mode 100644 index 4ea4ede..0000000 --- a/sample/base.py +++ /dev/null @@ -1,23 +0,0 @@ -#/* -#* Licensed to the Apache Software Foundation (ASF) under one or more -#* contributor license agreements. See the NOTICE file distributed with -#* this work for additional information regarding copyright ownership. -#* The ASF licenses this file to You under the Apache License, Version 2.0 -#* (the "License"); you may not use this file except in compliance with -#* the License. You may obtain a copy of the License at -#* -#* http://www.apache.org/licenses/LICENSE-2.0 -#* -#* Unless required by applicable law or agreed to in writing, software -#* distributed under the License is distributed on an "AS IS" BASIS, -#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#* See the License for the specific language governing permissions and -#* limitations under the License. -#*/ - -import __init__ -from librocketmqclientpython import * - -def showClientVersion(): - print("Python Client Version:") - print(GetVersion()) diff --git a/sample/testConsumer.py b/sample/testConsumer.py deleted file mode 100644 index 03ca587..0000000 --- a/sample/testConsumer.py +++ /dev/null @@ -1,52 +0,0 @@ -#/* -#* Licensed to the Apache Software Foundation (ASF) under one or more -#* contributor license agreements. See the NOTICE file distributed with -#* this work for additional information regarding copyright ownership. -#* The ASF licenses this file to You under the Apache License, Version 2.0 -#* (the "License"); you may not use this file except in compliance with -#* the License. You may obtain a copy of the License at -#* -#* http://www.apache.org/licenses/LICENSE-2.0 -#* -#* Unless required by applicable law or agreed to in writing, software -#* distributed under the License is distributed on an "AS IS" BASIS, -#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#* See the License for the specific language governing permissions and -#* limitations under the License. -#*/ - -import base -import time -from librocketmqclientpython import * - -totalMsg = 0 - -def consumerMessage(msg, args): - global totalMsg - totalMsg += 1 - print(">>ConsumerMessage Called:",totalMsg) - print(GetMessageTopic(msg)) - print(GetMessageTags(msg)) - print(GetMessageBody(msg)) - print(GetMessageId(msg)) - return 0 - -print("Consumer Starting.....") - -consumer = CreatePushConsumer("awtTest_Producer_Python_Test") -print(consumer) -SetPushConsumerNameServerAddress(consumer, "172.17.0.2:9876") -SetPushConsumerThreadCount(consumer, 1) -Subscribe(consumer, "T_TestTopic", "*") -RegisterMessageCallback(consumer, consumerMessage, None) -StartPushConsumer(consumer) - -i = 1 -while i <= 60: - print(i) - i += 1 - time.sleep(10) - -ShutdownPushConsumer(consumer) -DestroyPushConsumer(consumer) -print("Consumer Down....") diff --git a/sample/testProducer.py b/sample/testProducer.py deleted file mode 100644 index 34023c6..0000000 --- a/sample/testProducer.py +++ /dev/null @@ -1,100 +0,0 @@ -# /* -# * Licensed to the Apache Software Foundation (ASF) under one or more -# * contributor license agreements. See the NOTICE file distributed with -# * this work for additional information regarding copyright ownership. -# * The ASF licenses this file to You under the Apache License, Version 2.0 -# * (the "License"); you may not use this file except in compliance with -# * the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -from base import * -import time - - -def initProducer(name): - print("---------Create Producer---------------") - producer = CreateProducer(name) - SetProducerNameServerAddress(producer, "172.17.0.2:9876") - StartProducer(producer) - return producer - - -def testSendMssage(producer, topic, key, body): - print("Starting Sending.....") - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - SetMessageTags(msg, "ThisMessageTag.") - result = SendMessageSync(producer, msg) - print(result) - print("Msgid:") - print(result.GetMsgId()) - print("Offset:") - print(result.offset) - print("sendStatus:") - print(result.sendStatus) - DestroyMessage(msg) - print("Done...............") - - -def testSendMessageOneway(producer, topic, key, body): - print("Starting Sending(Oneway).....") - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - SetMessageTags(msg, "Send Message Oneway Test.") - SendMessageOneway(producer, msg) - DestroyMessage(msg) - print("Done...............") - - -def testSendMssageOrderly(producer, topic, key, body): - print("Starting Sending.....") - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - SetMessageTags(msg, "ThisMessageTag.") - result = SendMessageOrderlyByShardingKey(producer, msg, "orderId") - print(result) - print("Msgid:") - print(result.GetMsgId()) - print("Offset:") - print(result.offset) - print("sendStatus:") - print(result.sendStatus) - DestroyMessage(msg) - print("Done...............") - - -def releaseProducer(producer): - ShutdownProducer(producer) - DestroyProducer(producer) - print("--------Release producer-----------") - - -showClientVersion() -producer = initProducer("TestPythonProducer") -topic = "T_TestTopic" -key = "TestKeys" -body = "ThisIsTestBody" -i = 0 -while i < 100: - i += 1 - testSendMssageOrderly(producer, topic, key, body) - - print("Now Send Message:", i) - -while i < 10: - i += 1 - testSendMessageOneway(producer, topic, key, body) - print("Now Send Message One way:", i) - -releaseProducer(producer) diff --git a/samples/__init__.py b/samples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/samples/consumer.py b/samples/consumer.py new file mode 100644 index 0000000..bf36c4a --- /dev/null +++ b/samples/consumer.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from rocketmq.client import PushConsumer, ConsumeStatus +import time + +def callback(msg): + print(msg.id, msg.body, msg.get_property('property')) + return ConsumeStatus.CONSUME_SUCCESS + +def start_consume_message(): + consumer = PushConsumer('consumer_group') + consumer.set_name_server_address('127.0.0.1:9876') + consumer.subscribe('BenchmarkTest', callback) + # consumer.set_ssl_enable(True) + # consumer.set_ssl_property_file("/etc/rocketmq/tls.properties") + print ('start consume message') + consumer.start() + + while True: + time.sleep(3600) + +if __name__ == '__main__': + start_consume_message() \ No newline at end of file diff --git a/samples/producer.py b/samples/producer.py new file mode 100644 index 0000000..4869b77 --- /dev/null +++ b/samples/producer.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from rocketmq.client import Producer, Message, TransactionMQProducer, TransactionStatus + +import time +import threading + +topic = 'TopicTest' +gid = 'test' +name_srv = '127.0.0.1:9876' +MUTEX = threading.Lock() + +def create_message(): + msg = Message(topic) + msg.set_keys('XXX') + msg.set_tags('XXX') + msg.set_property('property', 'test') + msg.set_body('message body') + return msg + + +def send_message_sync(count): + producer = Producer(gid) + producer.set_name_server_address(name_srv) + producer.start() + for n in range(count): + msg = create_message() + ret = producer.send_sync(msg) + print ('send message status: ' + str(ret.status) + ' msgId: ' + ret.msg_id) + print ('send sync message done') + producer.shutdown() + + +def send_message_multi_threaded(retry_time): + producer = Producer(gid) + producer.set_name_server_address(name_srv) + msg = create_message() + + global MUTEX + MUTEX.acquire() + try: + producer.start() + except Exception as e: + print('ProducerStartFailed:', e) + MUTEX.release() + return + + try: + for i in range(retry_time): + ret = producer.send_sync(msg) + if ret.status == 0: + print('send message status: ' + str(ret.status) + ' msgId: ' + ret.msg_id) + break + else: + print('send message to MQ failed.') + if i == (retry_time - 1): + print('send message to MQ failed after retries.') + except Exception as e: + print('ProducerSendSyncFailed:', e) + finally: + producer.shutdown() + MUTEX.release() + return + + +def send_orderly_with_sharding_key(count): + producer = Producer(gid, True) + producer.set_name_server_address(name_srv) + producer.start() + for n in range(count): + msg = create_message() + ret = producer.send_orderly_with_sharding_key(msg, 'orderId') + print ('send message status: ' + str(ret.status) + ' msgId: ' + ret.msg_id) + print ('send sync order message done') + producer.shutdown() + + +def check_callback(msg): + print ('check: ' + msg.body.decode('utf-8')) + return TransactionStatus.COMMIT + + +def local_execute(msg, user_args): + print ('local: ' + msg.body.decode('utf-8')) + return TransactionStatus.UNKNOWN + + +def send_transaction_message(count): + producer = TransactionMQProducer(gid, check_callback) + producer.set_name_server_address(name_srv) + producer.start() + for n in range(count): + msg = create_message() + ret = producer.send_message_in_transaction(msg, local_execute, None) + print ('send message status: ' + str(ret.status) + ' msgId: ' + ret.msg_id) + print ('send transaction message done') + + while True: + time.sleep(3600) + + +def send_message_with_ssl(count): + producer = Producer(gid) + producer.set_name_server_address(name_srv) + producer.set_ssl_enable(True) + producer.set_ssl_property_file("/etc/rocketmq/tls.properties") + producer.start() + for n in range(count): + msg = create_message() + producer.start() + for n in range(count): + msg = create_message() + ret = producer.send_sync(msg) + print ('send message status: ' + str(ret.status) + ' msgId: ' + ret.msg_id) + print ('send sync message done') + producer.shutdown() + + +if __name__ == '__main__': + send_message_sync(10) + diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..85298f9 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +[aliases] +release = sdist bdist_wheel + +[bdist_wheel] +universal = 0 + +[flake8] +exclude = .svn,CVS,.bzr,.hg,.git,__pycache,.ropeproject +max-line-length = 120 +import-order-style = pep8 +application-import-names = rocketmq diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index a7d262d..7bdcf81 --- a/setup.py +++ b/setup.py @@ -1,46 +1,91 @@ -# /* -# * Licensed to the Apache Software Foundation (ASF) under one or more -# * contributor license agreements. See the NOTICE file distributed with -# * this work for additional information regarding copyright ownership. -# * The ASF licenses this file to You under the Apache License, Version 2.0 -# * (the "License"); you may not use this file except in compliance with -# * the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -from distutils.core import Extension - -from setuptools import setup - -BOOST_INCLUDE_PATH = "/usr/local/include/boost" -PYTHON_INCLUDE_PATH = "/usr/include/python2.7" -ROCKETMQ_INCLUDE_PATH = "/usr/local/include/rocketmq" -PYTHON_LIB_DIR = "/usr/lib64" -BOOST_LIB_DIR = "/usr/local/lib" -ROCKETMQ_LIB_DIR = "/usr/local/lib" -NAME = 'librocketmqclientpython' -setup(name=NAME, - version='1.2.0', - url="https://github.com/apache/rocketmq-client-python", - description="RocketMQ Python client", - long_description="RocketMQ Python client is developed on top of rocketmq-client-cpp, which has been proven " - "robust and widely adopted within Alibaba Group by many business units for more than three " - "years.", - license="Apache License, Version 2.0", - platforms=["linux"], - packages=["src"], - ext_modules=[Extension(name=NAME - , sources=['src/PythonWrapper.cpp'] - , extra_compile_args=[] - , extra_link_args=["-lboost_python", "-lrocketmq"] - , include_dirs=[BOOST_INCLUDE_PATH, ROCKETMQ_INCLUDE_PATH] - , library_dirs=[PYTHON_LIB_DIR, ROCKETMQ_LIB_DIR, BOOST_LIB_DIR] - ), ], - ) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import os +import sys +import struct + +from setuptools import setup, find_packages +from setuptools.command.install import install + +readme = 'README.md' +with open(readme) as f: + long_description = f.read() + +# from https://stackoverflow.com/questions/45150304/how-to-force-a-python-wheel-to-be-platform-specific-when-building-it # noqa +cmdclass = {} +try: + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel + + + class bdist_wheel(_bdist_wheel): + def finalize_options(self): + _bdist_wheel.finalize_options(self) + # Mark us as not a pure python package (we have platform specific C/C++ code) + self.root_is_pure = False + + def get_tag(self): + # this set's us up to build generic wheels. + python, abi, plat = _bdist_wheel.get_tag(self) + python, abi = 'py2.py3', 'none' + return python, abi, plat + + + cmdclass['bdist_wheel'] = bdist_wheel + +except ImportError: + pass + + +class InstallPlatlib(install): + def finalize_options(self): + install.finalize_options(self) + # force platlib + self.install_lib = self.install_platlib + + +cmdclass['install'] = InstallPlatlib + +setup( + name='rocketmq-client-python', + version='2.0.1rc1', + author='apache.rocketmq', + author_email='dev@rocketmq.apache.org', + packages=find_packages(exclude=('tests', 'tests.*')), + keywords='rocketmq', + description='RocketMQ Python Client', + long_description=long_description, + long_description_content_type='text/markdown', + include_package_data=True, + install_requires=[ + "enum34; python_version<='3.4'", + ], + cmdclass=cmdclass, + classifiers=[ + 'Operating System :: MacOS', + 'Operating System :: POSIX', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + ] +) diff --git a/src/PythonWrapper.cpp b/src/PythonWrapper.cpp deleted file mode 100644 index 5fb1d53..0000000 --- a/src/PythonWrapper.cpp +++ /dev/null @@ -1,533 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "CCommon.h" -#include "CMessage.h" -#include "CMessageExt.h" -#include "CSendResult.h" -#include "CProducer.h" -#include "CPushConsumer.h" -#include "PythonWrapper.h" -#include "CMQException.h" -#include -#include - -using namespace boost::python; -using namespace std; - -const char *VERSION = - "PYTHON_CLIENT_VERSION: " PYTHON_CLIENT_VERSION ", BUILD DATE: " PYCLI_BUILD_DATE " "; - -map> g_CallBackMap; -map g_TransactionCheckCallBackMap; - - -class PyThreadStateLock { -public: - PyThreadStateLock() { - state = PyGILState_Ensure(); - } - - ~PyThreadStateLock() { - // NOTE: must paired with PyGILState_Ensure, otherwise it will cause deadlock!!! - PyGILState_Release(state); - } - -private: - PyGILState_STATE state; -}; - -class PyThreadStateUnlock { -public: - PyThreadStateUnlock() : _save(NULL) { - Py_UNBLOCK_THREADS - } - - ~PyThreadStateUnlock() { - Py_BLOCK_THREADS - } - -private: - PyThreadState *_save; -}; - -#ifdef __cplusplus -extern "C" { -#endif -void *PyCreateMessage(const char *topic) { - - return (void *) CreateMessage(topic); -} - -int PyDestroyMessage(void *msg) { - return DestroyMessage((CMessage *) msg); -} -int PySetMessageTopic(void *msg, const char *topic) { - return SetMessageTopic((CMessage *) msg, topic); -} -int PySetMessageTags(void *msg, const char *tags) { - return SetMessageTags((CMessage *) msg, tags); -} -int PySetMessageKeys(void *msg, const char *keys) { - return SetMessageKeys((CMessage *) msg, keys); -} -int PySetMessageBody(void *msg, const char *body) { - return SetMessageBody((CMessage *) msg, body); -} -int PySetByteMessageBody(void *msg, const char *body, int len) { - return SetByteMessageBody((CMessage *) msg, body, len); -} -int PySetMessageProperty(void *msg, const char *key, const char *value) { - return SetMessageProperty((CMessage *) msg, key, value); -} -int PySetMessageDelayTimeLevel(void *msg, int level) { - return SetDelayTimeLevel((CMessage *) msg, level); -} - - -//messageExt -const char *PyGetMessageTopic(PyMessageExt msgExt) { - return GetMessageTopic((CMessageExt *) msgExt.pMessageExt); -} -const char *PyGetMessageTags(PyMessageExt msgExt) { - return GetMessageTags((CMessageExt *) msgExt.pMessageExt); -} -const char *PyGetMessageKeys(PyMessageExt msgExt) { - return GetMessageKeys((CMessageExt *) msgExt.pMessageExt); -} -const char *PyGetMessageBody(PyMessageExt msgExt) { - return GetMessageBody((CMessageExt *) msgExt.pMessageExt); -} -const char *PyGetMessageProperty(PyMessageExt msgExt, const char *key) { - return GetMessageProperty((CMessageExt *) msgExt.pMessageExt, key); -} -const char *PyGetMessageId(PyMessageExt msgExt) { - return GetMessageId((CMessageExt *) msgExt.pMessageExt); -} - -//producer -void *PyCreateProducer(const char *groupId) { - PyEval_InitThreads(); // ensure create GIL, for call Python callback from C. - return (void *) CreateProducer(groupId); -} - -void *PyCreateOrderProducer(const char *groupId) { - PyEval_InitThreads(); // ensure create GIL, for call Python callback from C. - return (void *) CreateOrderlyProducer(groupId); -} - -void *PyCreateTransactionProducer(const char *groupId, PyObject *localTransactionCheckerCallback) { - PyEval_InitThreads(); - CProducer *producer = CreateTransactionProducer(groupId, &PyLocalTransactionCheckerCallback, NULL); - g_TransactionCheckCallBackMap[producer] = localTransactionCheckerCallback; - return producer; -} - -CTransactionStatus PyLocalTransactionCheckerCallback(CProducer *producer, CMessageExt *msg, void *data) { - PyThreadStateLock pyThreadLock; // ensure hold GIL, before call python callback - PyMessageExt message = {.pMessageExt = msg}; - map::iterator iter; - iter = g_TransactionCheckCallBackMap.find(producer); - if (iter != g_TransactionCheckCallBackMap.end()) { - PyObject *pCallback = iter->second; - CTransactionStatus status = boost::python::call(pCallback, message); - return status; - } - return CTransactionStatus::E_UNKNOWN_TRANSACTION; -} - -int PyDestroyProducer(void *producer) { - return DestroyProducer((CProducer *) producer); -} -int PyStartProducer(void *producer) { - return StartProducer((CProducer *) producer); -} -int PyShutdownProducer(void *producer) { - PyThreadStateUnlock PyThreadUnlock; // Shutdown Producer is a block call, ensure thread don't hold GIL. - return ShutdownProducer((CProducer *) producer); -} -int PySetProducerNameServerAddress(void *producer, const char *namesrv) { - return SetProducerNameServerAddress((CProducer *) producer, namesrv); -} -int PySetProducerNameServerDomain(void *producer, const char *domain) { - return SetProducerNameServerDomain((CProducer *) producer, domain); -} -int PySetProducerInstanceName(void *producer, const char *instanceName) { - return SetProducerInstanceName((CProducer *)producer, instanceName); -} -int PySetProducerSessionCredentials(void *producer, const char *accessKey, const char *secretKey, const char *channel) { - return SetProducerSessionCredentials((CProducer *)producer, accessKey, secretKey, channel); -} -int PySetProducerCompressLevel(void *producer, int level) { - return SetProducerCompressLevel((CProducer *)producer, level); -} -int PySetProducerMaxMessageSize(void *producer, int size) { - return SetProducerMaxMessageSize((CProducer *)producer, size); -} -int PySetProducerLogPath(void *producer, const char *logPath) { - return SetProducerLogPath((CProducer *) producer, logPath); -} -int PySetProducerLogFileNumAndSize(void *producer, int fileNum, long fileSize) { - return SetProducerLogFileNumAndSize((CProducer *) producer, fileNum, fileSize); -} -int PySetProducerLogLevel(void *producer, CLogLevel level) { - return SetProducerLogLevel((CProducer *) producer, level); -} -int PySetProducerSendMsgTimeout(void *producer, int timeout) { - return SetProducerSendMsgTimeout((CProducer *) producer, timeout); -} - -PySendResult PySendMessageSync(void *producer, void *msg) { - PySendResult ret; - CSendResult result; - SendMessageSync((CProducer *) producer, (CMessage *) msg, &result); - ret.sendStatus = result.sendStatus; - ret.offset = result.offset; - strncpy(ret.msgId, result.msgId, MAX_MESSAGE_ID_LENGTH - 1); - ret.msgId[MAX_MESSAGE_ID_LENGTH - 1] = 0; - return ret; -} - -int PySendMessageOneway(void *producer, void *msg) { - return SendMessageOneway((CProducer *) producer, (CMessage *) msg); -} - -void PySendSuccessCallback(CSendResult result, CMessage *msg, void *pyCallback) { - PyThreadStateLock PyThreadLock; // ensure hold GIL, before call python callback - PySendResult sendResult; - sendResult.sendStatus = result.sendStatus; - sendResult.offset = result.offset; - strncpy(sendResult.msgId, result.msgId, MAX_MESSAGE_ID_LENGTH - 1); - sendResult.msgId[MAX_MESSAGE_ID_LENGTH - 1] = 0; - PyCallback *callback = (PyCallback *) pyCallback; - boost::python::call(callback->successCallback, sendResult, (void *) msg); - delete pyCallback; -} - - -void PySendExceptionCallback(CMQException e, CMessage *msg, void *pyCallback) { - PyThreadStateLock PyThreadLock; // ensure hold GIL, before call python callback - PyMQException exception; - PyCallback *callback = (PyCallback *) pyCallback; - exception.error = e.error; - exception.line = e.line; - strncpy(exception.file, e.file, MAX_EXEPTION_FILE_LENGTH - 1); - exception.file[MAX_EXEPTION_FILE_LENGTH - 1] = 0; - strncpy(exception.msg, e.msg, MAX_EXEPTION_MSG_LENGTH - 1); - exception.msg[MAX_EXEPTION_MSG_LENGTH - 1] = 0; - strncpy(exception.type, e.type, MAX_EXEPTION_TYPE_LENGTH - 1); - exception.type[MAX_EXEPTION_TYPE_LENGTH - 1] = 0; - boost::python::call(callback->exceptionCallback, (void *) msg, exception); - delete pyCallback; -} - -int PySendMessageAsync(void *producer, void *msg, PyObject *sendSuccessCallback, PyObject *sendExceptionCallback) { - PyCallback *pyCallback = new PyCallback(); - pyCallback->successCallback = sendSuccessCallback; - pyCallback->exceptionCallback = sendExceptionCallback; - return SendAsync((CProducer *) producer, (CMessage *) msg, &PySendSuccessCallback, &PySendExceptionCallback, - (void *) pyCallback); -} - - -PySendResult PySendMessageOrderly(void *producer, void *msg, int autoRetryTimes, void *args, PyObject *queueSelector) { - PySendResult ret; - CSendResult result; - PyUserData userData = {queueSelector, args}; - SendMessageOrderly((CProducer *) producer, (CMessage *) msg, &PyOrderlyCallbackInner, &userData, autoRetryTimes, - &result); - ret.sendStatus = result.sendStatus; - ret.offset = result.offset; - strncpy(ret.msgId, result.msgId, MAX_MESSAGE_ID_LENGTH - 1); - ret.msgId[MAX_MESSAGE_ID_LENGTH - 1] = 0; - return ret; -} - -int PyOrderlyCallbackInner(int size, CMessage *msg, void *args) { - PyUserData *userData = (PyUserData *) args; - int index = boost::python::call(userData->pyObject, size, (void *) msg, userData->pData); - return index; -} - -PySendResult PySendMessageOrderlyByShardingKey(void *producer, void *msg, const char *shardingKey) { - PySendResult ret; - CSendResult result; - SendMessageOrderlyByShardingKey((CProducer *) producer, (CMessage *) msg, shardingKey, &result); - ret.sendStatus = result.sendStatus; - ret.offset = result.offset; - strncpy(ret.msgId, result.msgId, MAX_MESSAGE_ID_LENGTH - 1); - ret.msgId[MAX_MESSAGE_ID_LENGTH - 1] = 0; - return ret; -} - -CTransactionStatus PyLocalTransactionExecuteCallback(CProducer *producer, CMessage *msg, void *data) { - PyUserData *localCallback = (PyUserData *) data; - CTransactionStatus status = boost::python::call(localCallback->pyObject, (void *) msg, - localCallback->pData); - return status; -} - -PySendResult PySendMessageInTransaction(void *producer, void *msg, PyObject *localTransactionCallback, void *args) { - PyUserData userData = {localTransactionCallback, args}; - PySendResult ret; - CSendResult result; - SendMessageTransaction((CProducer *) producer, (CMessage *) msg, &PyLocalTransactionExecuteCallback, &userData, - &result); - ret.sendStatus = result.sendStatus; - ret.offset = result.offset; - strncpy(ret.msgId, result.msgId, MAX_MESSAGE_ID_LENGTH - 1); - ret.msgId[MAX_MESSAGE_ID_LENGTH - 1] = 0; - return ret; -} - -//SendResult -const char *PyGetSendResultMsgID(CSendResult &sendResult) { - return (const char *) (sendResult.msgId); -} -//consumer -void *PyCreatePushConsumer(const char *groupId) { - PyEval_InitThreads(); // ensure create GIL, for call Python callback from C. - return (void *) CreatePushConsumer(groupId); -} -int PyDestroyPushConsumer(void *consumer) { - CPushConsumer *consumerInner = (CPushConsumer *) consumer; - map>::iterator iter; - iter = g_CallBackMap.find(consumerInner); - if (iter != g_CallBackMap.end()) { - UnregisterMessageCallback(consumerInner); - g_CallBackMap.erase(iter); - } - return DestroyPushConsumer(consumerInner); -} -int PyDestroyTransactionProducer(void *producer) { - CProducer *producerInner = (CProducer *) producer; - map::iterator iter; - iter = g_TransactionCheckCallBackMap.find(producerInner); - if (iter != g_TransactionCheckCallBackMap.end()) { - g_TransactionCheckCallBackMap.erase(iter); - } - return DestroyProducer(producerInner); -} -int PyStartPushConsumer(void *consumer) { - return StartPushConsumer((CPushConsumer *) consumer); -} -int PyShutdownPushConsumer(void *consumer) { - PyThreadStateUnlock PyThreadUnlock; // ShutdownPushConsumer is a block call, ensure thread don't hold GIL. - return ShutdownPushConsumer((CPushConsumer *) consumer); -} -int PySetPushConsumerNameServerAddress(void *consumer, const char *namesrv) { - return SetPushConsumerNameServerAddress((CPushConsumer *) consumer, namesrv); -} -int PySetPushConsumerNameServerDomain(void *consumer, const char *domain){ - return SetPushConsumerNameServerDomain((CPushConsumer *) consumer, domain); -} -int PySubscribe(void *consumer, const char *topic, const char *expression) { - return Subscribe((CPushConsumer *) consumer, topic, expression); -} -int PyRegisterMessageCallback(void *consumer, PyObject *pCallback, object args) { - CPushConsumer *consumerInner = (CPushConsumer *) consumer; - g_CallBackMap[consumerInner] = make_pair(pCallback, std::move(args)); - return RegisterMessageCallback(consumerInner, &PythonMessageCallBackInner); -} - -int PyRegisterMessageCallbackOrderly(void *consumer, PyObject *pCallback, object args) { - CPushConsumer *consumerInner = (CPushConsumer *) consumer; - g_CallBackMap[consumerInner] = make_pair(pCallback, std::move(args)); - return RegisterMessageCallbackOrderly(consumerInner, &PythonMessageCallBackInner); -} - -int PythonMessageCallBackInner(CPushConsumer *consumer, CMessageExt *msg) { - PyThreadStateLock PyThreadLock; // ensure hold GIL, before call python callback - PyMessageExt message = { .pMessageExt = msg }; - map>::iterator iter; - iter = g_CallBackMap.find(consumer); - if (iter != g_CallBackMap.end()) { - pair callback = iter->second; - PyObject * pCallback = callback.first; - object& args = callback.second; - if (pCallback != NULL) { - int status = boost::python::call(pCallback, message, args); - return status; - } - } - return 1; -} - -int PySetPushConsumerThreadCount(void *consumer, int threadCount) { - return SetPushConsumerThreadCount((CPushConsumer *) consumer, threadCount); -} -int PySetPushConsumerMessageBatchMaxSize(void *consumer, int batchSize) { - return SetPushConsumerMessageBatchMaxSize((CPushConsumer *) consumer, batchSize); -} -int PySetPushConsumerInstanceName(void *consumer, const char *instanceName){ - return SetPushConsumerInstanceName((CPushConsumer *)consumer, instanceName); -} -int PySetPushConsumerSessionCredentials(void *consumer, const char *accessKey, const char *secretKey, - const char *channel){ - return SetPushConsumerSessionCredentials((CPushConsumer *)consumer, accessKey, secretKey, channel); -} -int PySetPushConsumerMessageModel(void *consumer, CMessageModel messageModel) { - return SetPushConsumerMessageModel((CPushConsumer *) consumer, messageModel); -} - -int PySetPushConsumerLogPath(void *consumer, const char *logPath) { - return SetPushConsumerLogPath((CPushConsumer *) consumer, logPath); -} - -int PySetPushConsumerLogFileNumAndSize(void *consumer, int fileNum, long fileSize) { - return SetPushConsumerLogFileNumAndSize((CPushConsumer *) consumer, fileNum, fileSize); -} - -int PySetPushConsumerLogLevel(void *consumer, CLogLevel level) { - return SetPushConsumerLogLevel((CPushConsumer *) consumer, level); -} - -//push consumer -int PySetPullConsumerNameServerDomain(void *consumer, const char *domain) { - return SetPullConsumerNameServerDomain((CPullConsumer *) consumer, domain); -} -//version -const char *PyGetVersion() { - return VERSION; -} -#ifdef __cplusplus -}; -#endif -BOOST_PYTHON_MODULE (librocketmqclientpython) { -/* - class_("CMessage"); - class_("CMessageExt"); - class_("CProducer"); - class_("CPushConsumer"); -*/ - enum_("CStatus") - .value("OK", OK) - .value("NULL_POINTER", NULL_POINTER); - - enum_("CSendStatus") - .value("E_SEND_OK", E_SEND_OK) - .value("E_SEND_FLUSH_DISK_TIMEOUT", E_SEND_FLUSH_DISK_TIMEOUT) - .value("E_SEND_FLUSH_SLAVE_TIMEOUT", E_SEND_FLUSH_SLAVE_TIMEOUT) - .value("E_SEND_SLAVE_NOT_AVAILABLE", E_SEND_SLAVE_NOT_AVAILABLE); - - enum_("CConsumeStatus") - .value("E_CONSUME_SUCCESS", E_CONSUME_SUCCESS) - .value("E_RECONSUME_LATER", E_RECONSUME_LATER); - - class_("SendResult") - .def_readonly("offset", &PySendResult::offset, "offset") - //.def_readonly("msgId", &PySendResult::msgId, "msgId") - .def_readonly("sendStatus", &PySendResult::sendStatus, "sendStatus") - .def("GetMsgId", &PySendResult::GetMsgId); - class_("CMessageExt"); - - class_("MQException") - .def_readonly("error", &PyMQException::error, "error") - .def_readonly("line", &PyMQException::line, "line") - .def("GetFile", &PyMQException::GetFile) - .def("GetMsg", &PyMQException::GetMsg) - .def("GetType", &PyMQException::GetType); - enum_("CMessageModel") - .value("BROADCASTING", BROADCASTING) - .value("CLUSTERING", CLUSTERING); - - enum_("CLogLevel") - .value("E_LOG_LEVEL_FATAL", E_LOG_LEVEL_FATAL) - .value("E_LOG_LEVEL_ERROR", E_LOG_LEVEL_ERROR) - .value("E_LOG_LEVEL_WARN", E_LOG_LEVEL_WARN) - .value("E_LOG_LEVEL_INFO", E_LOG_LEVEL_INFO) - .value("E_LOG_LEVEL_DEBUG", E_LOG_LEVEL_DEBUG) - .value("E_LOG_LEVEL_TRACE", E_LOG_LEVEL_TRACE) - .value("E_LOG_LEVEL_LEVEL_NUM", E_LOG_LEVEL_LEVEL_NUM); - - enum_("TransactionStatus") - .value("E_COMMIT_TRANSACTION", E_COMMIT_TRANSACTION) - .value("E_ROLLBACK_TRANSACTION", E_ROLLBACK_TRANSACTION) - .value("E_UNKNOWN_TRANSACTION", E_UNKNOWN_TRANSACTION); - - //For Message - def("CreateMessage", PyCreateMessage, return_value_policy()); - def("DestroyMessage", PyDestroyMessage); - def("SetMessageTopic", PySetMessageTopic); - def("SetMessageTags", PySetMessageTags); - def("SetMessageKeys", PySetMessageKeys); - def("SetMessageBody", PySetMessageBody); - def("SetByteMessageBody", PySetByteMessageBody); - def("SetMessageProperty", PySetMessageProperty); - def("SetDelayTimeLevel", PySetMessageDelayTimeLevel); - - //For MessageExt - def("GetMessageTopic", PyGetMessageTopic); - def("GetMessageTags", PyGetMessageTags); - def("GetMessageKeys", PyGetMessageKeys); - def("GetMessageBody", PyGetMessageBody); - def("GetMessageProperty", PyGetMessageProperty); - def("GetMessageId", PyGetMessageId); - - //For producer - def("CreateProducer", PyCreateProducer, return_value_policy()); - def("CreateOrderProducer", PyCreateOrderProducer, return_value_policy()); - def("CreateTransactionProducer", PyCreateTransactionProducer, return_value_policy()); - def("DestroyProducer", PyDestroyProducer); - def("DestroyTransactionProducer", PyDestroyTransactionProducer); - def("StartProducer", PyStartProducer); - def("ShutdownProducer", PyShutdownProducer); - def("SetProducerNameServerAddress", PySetProducerNameServerAddress); - def("SetProducerNameServerDomain", PySetProducerNameServerDomain); - def("SetProducerInstanceName", PySetProducerInstanceName); - def("SetProducerSessionCredentials", PySetProducerSessionCredentials); - def("SetProducerCompressLevel", PySetProducerCompressLevel); - def("SetProducerMaxMessageSize", PySetProducerMaxMessageSize); - def("SetProducerSendMsgTimeout", PySetProducerSendMsgTimeout); - - def("SetProducerLogPath", PySetProducerLogPath); - def("SetProducerLogFileNumAndSize", PySetProducerLogFileNumAndSize); - def("SetProducerLogLevel", PySetProducerLogLevel); - - def("SendMessageSync", PySendMessageSync); - def("SendMessageAsync", PySendMessageAsync); - - def("SendMessageOneway", PySendMessageOneway); - def("SendMessageOrderly", PySendMessageOrderly); - def("SendMessageOrderlyByShardingKey", PySendMessageOrderlyByShardingKey); - def("SendMessageInTransaction", PySendMessageInTransaction); - - //For Consumer - def("CreatePushConsumer", PyCreatePushConsumer, return_value_policy()); - def("DestroyPushConsumer", PyDestroyPushConsumer); - def("StartPushConsumer", PyStartPushConsumer); - def("ShutdownPushConsumer", PyShutdownPushConsumer); - def("SetPushConsumerNameServerAddress", PySetPushConsumerNameServerAddress); - def("SetPushConsumerNameServerDomain", PySetPushConsumerNameServerDomain); - def("SetPushConsumerThreadCount", PySetPushConsumerThreadCount); - def("SetPushConsumerMessageBatchMaxSize", PySetPushConsumerMessageBatchMaxSize); - def("SetPushConsumerInstanceName", PySetPushConsumerInstanceName); - def("SetPushConsumerSessionCredentials", PySetPushConsumerSessionCredentials); - def("Subscribe", PySubscribe); - def("RegisterMessageCallback", PyRegisterMessageCallback); - def("RegisterMessageCallbackOrderly", PyRegisterMessageCallbackOrderly); - def("SetPushConsumerLogPath", PySetPushConsumerLogPath); - def("SetPushConsumerLogFileNumAndSize", PySetPushConsumerLogFileNumAndSize); - def("SetPushConsumerLogLevel", PySetPushConsumerLogLevel); - - //pull consumer - def("SetPullConsumerNameServerDomain", PySetPullConsumerNameServerDomain); - def("SetPushConsumerMessageModel", PySetPushConsumerMessageModel); - - //For Version - def("GetVersion", PyGetVersion); -} - diff --git a/src/PythonWrapper.h b/src/PythonWrapper.h deleted file mode 100644 index 8fdb04f..0000000 --- a/src/PythonWrapper.h +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "CCommon.h" -#include "CMessage.h" -#include "CMessageExt.h" -#include "CSendResult.h" -#include "CProducer.h" -#include "CPushConsumer.h" -#include "CPullConsumer.h" -#include "CMQException.h" -#include - -using namespace boost::python; - -typedef struct _PySendResult_ { - CSendStatus sendStatus; - char msgId[MAX_MESSAGE_ID_LENGTH]; - long long offset; - - const char *GetMsgId() { - return (const char *) msgId; - } -} PySendResult; - -typedef struct _PyMQException_ { - int error; - int line; - char file[MAX_EXEPTION_FILE_LENGTH]; - char msg[MAX_EXEPTION_MSG_LENGTH]; - char type[MAX_EXEPTION_TYPE_LENGTH]; - - const char *GetFile() { - return (const char *) file; - } - - const char *GetMsg() { - return (const char *) msg; - } - - const char *GetType() { - return (const char *) type; - } -} PyMQException; - - -typedef struct _PyMessageExt_ { - CMessageExt *pMessageExt; -} PyMessageExt; - -typedef struct _PyUserData_ { - PyObject *pyObject; - void *pData; -} PyUserData; - -typedef struct _PyCallback_ { - PyObject *successCallback; - PyObject *exceptionCallback; -} PyCallback; - -#define PYTHON_CLIENT_VERSION "1.2.0" -#define PYCLI_BUILD_DATE "04-12-2018" - -#ifdef __cplusplus -extern "C" { -#endif - -//message -void *PyCreateMessage(const char *topic); -int PyDestroyMessage(void *msg); -int PySetMessageTopic(void *msg, const char *topic); -int PySetMessageTags(void *msg, const char *tags); -int PySetMessageKeys(void *msg, const char *keys); -int PySetMessageBody(void *msg, const char *body); -int PySetByteMessageBody(void *msg, const char *body, int len); -int PySetMessageProperty(void *msg, const char *key, const char *value); -int PySetMessageDelayTimeLevel(void *msg, int level); - -//messageExt -const char *PyGetMessageTopic(PyMessageExt msgExt); -const char *PyGetMessageTags(PyMessageExt msgExt); -const char *PyGetMessageKeys(PyMessageExt msgExt); -const char *PyGetMessageBody(PyMessageExt msgExt); -const char *PyGetMessageProperty(PyMessageExt msgExt, const char *key); -const char *PyGetMessageId(PyMessageExt msgExt); - -//producer -void *PyCreateProducer(const char *groupId); -void *PyCreateOrderProducer(const char *groupId); -CTransactionStatus PyLocalTransactionCheckerCallback(CProducer *producer, CMessageExt *msg, void *data); -CTransactionStatus PyLocalTransactionExecuteCallback(CProducer *producer, CMessage *msg, void *data); -void *PyCreateTransactionProducer(const char *groupId, PyObject *localTransactionCheckerCallback); - -int PyDestroyProducer(void *producer); -int PyDestroyTransactionProducer(void *producer); -int PyStartProducer(void *producer); -int PyShutdownProducer(void *producer); -int PySetProducerNameServerAddress(void *producer, const char *namesrv); -int PySetProducerNameServerDomain(void *producer, const char *domain); -int PySetProducerInstanceName(void *producer, const char *instanceName); -int PySetProducerSessionCredentials(void *producer, const char *accessKey, const char *secretKey, const char *channel); -int PySetProducerCompressLevel(void *producer, int level); -int PySetProducerMaxMessageSize(void *producer, int size); -int PySetProducerLogPath(void *producer, const char *logPath); -int PySetProducerLogFileNumAndSize(void *producer, int fileNum, long fileSize); -int PySetProducerLogLevel(void *producer, CLogLevel level); -int PySetProducerSendMsgTimeout(void *producer, int timeout); - -PySendResult PySendMessageSync(void *producer, void *msg); -int PySendMessageOneway(void *producer, void *msg); - -void PySendSuccessCallback(CSendResult result, CMessage *msg, void *pyCallback); -void PySendExceptionCallback(CMQException e, CMessage *msg, void *pyCallback); -int PySendMessageAsync(void *producer, void *msg, PyObject *sendSuccessCallback, PyObject *sendExceptionCallback); - -PySendResult PySendMessageOrderly(void *producer, void *msg, int autoRetryTimes, void *args, PyObject *queueSelector); -PySendResult PySendMessageOrderlyByShardingKey(void *producer, void *msg, const char *shardingKey); -PySendResult PySendMessageInTransaction(void *producer , void *msg, PyObject *localTransactionExecuteCallback , void *args); - -int PyOrderlyCallbackInner(int size, CMessage *msg, void *args); - -//sendResult -const char *PyGetSendResultMsgID(CSendResult &sendResult); - -//consumer -void *PyCreatePushConsumer(const char *groupId); -int PyDestroyPushConsumer(void *consumer); -int PyStartPushConsumer(void *consumer); -int PyShutdownPushConsumer(void *consumer); -int PySetPushConsumerNameServerAddress(void *consumer, const char *namesrv); -int PySetPushConsumerNameServerDomain(void *consumer, const char *domain); -int PySubscribe(void *consumer, const char *topic, const char *expression); -int PyRegisterMessageCallback(void *consumer, PyObject *pCallback, object args); -int PyRegisterMessageCallbackOrderly(void *consumer, PyObject *pCallback, object args); -int PythonMessageCallBackInner(CPushConsumer *consumer, CMessageExt *msg); -int PySetPushConsumerThreadCount(void *consumer, int threadCount); -int PySetPushConsumerMessageBatchMaxSize(void *consumer, int batchSize); -int PySetPushConsumerInstanceName(void *consumer, const char *instanceName); -int PySetPushConsumerSessionCredentials(void *consumer, const char *accessKey, const char *secretKey, const char *channel); -int PySetPushConsumerMessageModel(void *consumer, CMessageModel messageModel); -int PySetPushConsumerLogPath(void *consumer, const char *logPath); -int PySetPushConsumerLogFileNumAndSize(void *consumer, int fileNum, long fileSize); -int PySetPushConsumerLogLevel(void *consumer, CLogLevel level); - -//push consumer -int PySetPullConsumerNameServerDomain(void *consumer, const char *domain); -//version -const char *PyGetVersion(); - -#ifdef __cplusplus -}; -#endif - diff --git a/test/TestConsumeMessages.py b/test/TestConsumeMessages.py deleted file mode 100644 index c3c252a..0000000 --- a/test/TestConsumeMessages.py +++ /dev/null @@ -1,94 +0,0 @@ -# /* -# * Licensed to the Apache Software Foundation (ASF) under one or more -# * contributor license agreements. See the NOTICE file distributed with -# * this work for additional information regarding copyright ownership. -# * The ASF licenses this file to You under the Apache License, Version 2.0 -# * (the "License"); you may not use this file except in compliance with -# * the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -import __init__ -from librocketmqclientpython import * - -import time -import sys - -topic = 'test-topic-normal' -topic_orderly = 'test-topic-normal-orderly' - -name_srv = '127.0.0.1:9876' -tag = 'rmq-tag' -consumer_group = 'test-consumer-group' -consumer_group_orderly = 'test-topic-normal-orderly_group' -totalMsg = 0 - - -def sigint_handler(signum, frame): - global is_sigint_up - is_sigint_up = True - sys.exit(0) - - -def consumer_message(msg, args): - global totalMsg - totalMsg += 1 - print 'total count %d' % totalMsg - print 'topic=%s' % GetMessageTopic(msg) - print 'tag=%s' % GetMessageTags(msg) - print 'body=%s' % GetMessageBody(msg) - print 'msg id=%s' % GetMessageId(msg) - - print 'map.keys %s' % GetMessageKeys(msg) - - print 'map.name %s' % GetMessageProperty(msg, 'name') - print 'map.id %s' % GetMessageProperty(msg, 'id') - return CConsumeStatus.E_CONSUME_SUCCESS - -def init_consumer(_group, _topic, _tag): - consumer = CreatePushConsumer(_group) - SetPushConsumerNameServerAddress(consumer, name_srv) - SetPushConsumerThreadCount(consumer, 1) - SetPushConsumerLogLevel(consumer, CLogLevel.E_LOG_LEVEL_INFO) - SetPushConsumerMessageModel(consumer, CMessageModel.CLUSTERING) - Subscribe(consumer, _topic, _tag) - RegisterMessageCallback(consumer, consumer_message, None) - StartPushConsumer(consumer) - print 'consumer is ready...' - return consumer - - -def start_one_consumer(_group, _topic, _tag): - consumer = init_consumer(_group, _topic, _tag) - i = 1 - while i <= 10: - print 'clock: ' + str(i) - i += 1 - time.sleep(10) - - ShutdownPushConsumer(consumer) - DestroyPushConsumer(consumer) - print("Consumer Down....") - -def start_orderly_consumer(): - consumer = init_consumer(consumer_group_orderly, topic_orderly, "*") - i = 1 - while i <= 10: - print 'clock: ' + str(i) - i += 1 - time.sleep(10) - - ShutdownPushConsumer(consumer) - DestroyPushConsumer(consumer) - print("Consumer Down....") - - -if __name__ == '__main__': - start_orderly_consumer() diff --git a/test/TestSendMessages.py b/test/TestSendMessages.py deleted file mode 100644 index 179e1f1..0000000 --- a/test/TestSendMessages.py +++ /dev/null @@ -1,290 +0,0 @@ -# /* -# * Licensed to the Apache Software Foundation (ASF) under one or more -# * contributor license agreements. See the NOTICE file distributed with -# * this work for additional information regarding copyright ownership. -# * The ASF licenses this file to You under the Apache License, Version 2.0 -# * (the "License"); you may not use this file except in compliance with -# * the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -import __init__ -from librocketmqclientpython import * -import time - -topic = 'test-topic-normal' -topic_orderly = 'test-topic-normal-orderly' -name_srv = '127.0.0.1:9876' - - -def init_producer(): - producer = CreateProducer('TestProducer') - SetProducerLogLevel(producer, CLogLevel.E_LOG_LEVEL_INFO) - SetProducerNameServerAddress(producer, name_srv) - StartProducer(producer) - return producer - -def transaction_local_checker(msg): - print 'begin check for msg: ' + GetMessageId(msg) - return TransactionStatus.E_COMMIT_TRANSACTION - -def init_transaction_producer(): - producer = CreateTransactionProducer('TransactionTestProducer', transaction_local_checker) - SetProducerLogLevel(producer, CLogLevel.E_LOG_LEVEL_INFO) - SetProducerNameServerAddress(producer, name_srv) - StartProducer(producer) - return producer - -producer = init_transaction_producer() -tag = 'rmq-tag' -key = 'rmq-key' - - -def send_messages_sync(count): - for a in range(count): - print 'start sending...' - body = 'hi rmq, now is ' + \ - time.strftime('%Y.%m.%d', time.localtime(time.time())) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - result = SendMessageSync(producer, msg) - DestroyMessage(msg) - print '[RMQ-PRODUCER]start sending...done, msg id = ' + \ - result.GetMsgId() - - -def send_messages_sync_with_map(count): - print 'sending message with properties...id, name' - for a in range(count): - body = 'hi rmq, now is ' + \ - time.strftime('%Y.%m.%d', time.localtime(time.time())) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - - SetMessageProperty(msg, 'name', 'test') - SetMessageProperty(msg, 'id', str(time.time())) - - result = SendMessageSync(producer, msg) - DestroyMessage(msg) - print '[RMQ-PRODUCER]start sending...done, msg id = ' + \ - result.GetMsgId() - - -def send_messages_with_tag_sync(count): - print 'sending message with tag...' + tag - for a in range(count): - body = 'hi rmq, now is ' + \ - time.strftime('%Y.%m.%d', time.localtime(time.time())) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageTags(msg, tag) - result = SendMessageSync(producer, msg) - DestroyMessage(msg) - print 'msg id = ' + result.GetMsgId() - - -def send_messages_with_tag_and_map_sync(count): - print 'sending message with tag...' + tag + ' and properties id, name' - for a in range(count): - body = 'hi rmq, now is ' + \ - time.strftime('%Y.%m.%d', time.localtime(time.time())) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - - SetMessageProperty(msg, 'name', 'test') - SetMessageProperty(msg, 'id', str(time.time())) - - SetMessageTags(msg, tag) - result = SendMessageSync(producer, msg) - DestroyMessage(msg) - print 'msg id = ' + result.GetMsgId() - - -def send_messages_with_key_sync(count): - print 'sending message with keys...' + key - for a in range(count): - body = 'hi rmq, now is ' + \ - time.strftime('%Y.%m.%d', time.localtime(time.time())) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - result = SendMessageSync(producer, msg) - DestroyMessage(msg) - print 'msg id = ' + result.GetMsgId() - - -def send_messages_with_key_and_map_sync(count): - print 'sending message with keys...' + key + ' and properties id, name' - for a in range(count): - body = 'hi rmq, now is ' + \ - time.strftime('%Y.%m.%d', time.localtime(time.time())) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - - SetMessageProperty(msg, 'name', 'test') - SetMessageProperty(msg, 'id', str(time.time())) - - result = SendMessageSync(producer, msg) - DestroyMessage(msg) - print 'msg id = ' + result.GetMsgId() - - -def send_messages_with_key_and_tag_sync(count): - key = 'rmq-key' - print 'sending message with keys and tag...' + key + ', ' + tag - for a in range(count): - body = 'hi rmq, now is ' + \ - time.strftime('%Y.%m.%d', time.localtime(time.time())) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - SetMessageTags(msg, tag) - result = SendMessageSync(producer, msg) - DestroyMessage(msg) - print 'msg id = ' + result.GetMsgId() - - -def send_messages_with_key_and_tag_and_map_sync(count): - key = 'rmq-key' - print 'sending message with keys and tag...' + \ - key + ', ' + tag + ' and properties id, name' - for a in range(count): - body = 'hi rmq, now is ' + \ - time.strftime('%Y.%m.%d', time.localtime(time.time())) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - - SetMessageProperty(msg, 'name', 'test') - SetMessageProperty(msg, 'id', str(time.time())) - - SetMessageTags(msg, tag) - result = SendMessageSync(producer, msg) - DestroyMessage(msg) - print 'msg id = ' + result.GetMsgId() - - -def send_messages_oneway(count): - for a in range(count): - print 'start sending...' - body = 'hi rmq, this is oneway message. now is ' + \ - time.strftime('%Y.%m.%d', time.localtime(time.time())) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - - SetMessageKeys(msg, key) - SetMessageProperty(msg, 'name', 'test') - SetMessageProperty(msg, 'id', str(time.time())) - - SendMessageOneway(producer, msg) - DestroyMessage(msg) - print 'send oneway is over' - - -def send_delay_messages(producer, topic, count): - key = 'rmq-key' - print 'start sending message' - tag = 'test' - for n in range(count): - body = 'hi rmq, now is' + str(time.time()) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - SetMessageProperty(msg, 'name', 'hello world') - SetMessageProperty(msg, 'id', str(time.time())) - SetMessageTags(msg, tag) - # messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h - - SetDelayTimeLevel(msg, 5) - - print str(msg) - result = SendMessageSync(producer, msg) - DestroyMessage(msg) - print 'msg id =' + result.GetMsgId() - -def send_message_orderly(count): - key = 'rmq-key' - print 'start sending order-ly message' - tag = 'test' - for n in range(count): - body = 'hi rmq orderly-message, now is' + str(n) - msg = CreateMessage(topic_orderly) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - SetMessageTags(msg, tag) - - result = SendMessageOrderly(producer, msg, 1, None, calc_which_queue_to_send) - DestroyMessage(msg) - print 'msg id =' + result.GetMsgId() - -def send_message_orderly_with_shardingkey(count): - key = 'rmq-key' - print 'start sending sharding key order-ly message' - tag = 'test' - for n in range(count): - body = 'hi rmq sharding orderly-message, now is' + str(n) - msg = CreateMessage(topic_orderly) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - SetMessageTags(msg, tag) - - result = SendMessageOrderlyByShardingKey(producer, msg, 'orderId') - DestroyMessage(msg) - print 'msg id =' + result.GetMsgId() - -def calc_which_queue_to_send(size, msg, arg): ## it is index start with 0.... - return 0 - -def send_message_async(count): - key = 'rmq-key' - print 'start sending message' - tag = 'test' - for n in range(count): - body = 'hi rmq message, now is' + str(n) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - SetMessageTags(msg, tag) - - SendMessageAsync(producer, msg, send_message_async_success, send_message_async_fail) - DestroyMessage(msg) - print 'send async message done' - time.sleep(10000) - -def send_message_async_success(result, msg): - print 'send success' - print 'msg id =' + result.GetMsgId() - -def send_message_async_fail(msg, exception): - print 'send message failed' - print 'error msg: ' + exception.GetMsg() - -def send_transaction_message(count): - key = 'rmq-key' - print 'start send transaction message' - tag = 'test' - for n in range(count): - body = 'hi rmq message, now is' + str(n) - msg = CreateMessage(topic) - SetMessageBody(msg, body) - SetMessageKeys(msg, key) - SetMessageTags(msg, tag) - - SendMessageInTransaction(producer, msg, transaction_local_execute, None) - print 'send transaction message done' - time.sleep(10000) - -def transaction_local_execute(msg, args): - print 'begin execute local transaction' - return TransactionStatus.E_UNKNOWN_TRANSACTION - -if __name__ == '__main__': - send_transaction_message(10) diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index f3a3a82..0000000 --- a/test/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# /* -# * Licensed to the Apache Software Foundation (ASF) under one or more -# * contributor license agreements. See the NOTICE file distributed with -# * this work for additional information regarding copyright ownership. -# * The ASF licenses this file to You under the Apache License, Version 2.0 -# * (the "License"); you may not use this file except in compliance with -# * the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -import sys -sys.path.append('/usr/local/lib') -print("__________Python Version:___________") -print(sys.version) -print("______Add Path /usr/local/lib_______") diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d649e12 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest +from rocketmq.client import Producer, PushConsumer + + +@pytest.fixture(scope='session') +def producer(): + prod = Producer('producer_group') + prod.set_name_server_address('127.0.0.1:9876') + prod.start() + yield prod + prod.shutdown() + + +@pytest.fixture(scope='session') +def orderly_producer(): + prod = Producer('orderly_producer_group', True) + prod.set_name_server_address('127.0.0.1:9876') + prod.start() + yield prod + prod.shutdown() + + +@pytest.fixture(scope='function') +def push_consumer(): + consumer = PushConsumer('push_consumer_group') + consumer.set_name_server_address('127.0.0.1:9876') + yield consumer + consumer.shutdown() diff --git a/tests/test_consumer.py b/tests/test_consumer.py new file mode 100644 index 0000000..e83d3c0 --- /dev/null +++ b/tests/test_consumer.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import time +import threading + +import pytest + +from rocketmq.client import Message, SendStatus, ConsumeStatus, PushConsumer +from rocketmq.exceptions import PushConsumerStartFailed + + +def _send_test_msg(producer): + msg = Message('test') + msg.set_keys('XXX') + msg.set_tags('XXX') + msg.set_body('XXXX') + msg.set_property('property', 'test') + ret = producer.send_sync(msg) + assert ret.status == SendStatus.OK + + +def test_push_consumer_no_subscription_start_fail(): + consumer = PushConsumer('testGroup') + consumer.set_name_server_address("127.0.0.1:9876") + with pytest.raises(PushConsumerStartFailed): + consumer.start() + + +def test_push_consumer(producer, push_consumer): + stop_event = threading.Event() + _send_test_msg(producer) + errors = [] + + def on_message(msg): + stop_event.set() + try: + assert msg.body.decode('utf-8') == 'XXXX' + assert msg.keys.decode('utf-8') == 'XXX' + assert msg.get_property('property').decode('utf-8') == 'test' + return ConsumeStatus.CONSUME_SUCCESS + except Exception as exc: + errors.append(exc) + return ConsumeStatus.RECONSUME_LATER + + push_consumer.subscribe('test', on_message) + push_consumer.start() + while not stop_event.is_set(): + time.sleep(2) + if errors: + raise errors[0] + + +def test_push_consumer_reconsume_later(producer, push_consumer): + stop_event = threading.Event() + _send_test_msg(producer) + raised_exc = threading.Event() + errors = [] + + def on_message(msg): + if not raised_exc.is_set(): + raised_exc.set() + return ConsumeStatus.RECONSUME_LATER + + stop_event.set() + try: + assert msg.body.decode('utf-8') == 'XXXX' + assert msg.keys.decode('utf-8') == 'XXX' + except Exception as exc: + errors.append(exc) + return ConsumeStatus.CONSUME_SUCCESS + + push_consumer.subscribe('test', on_message) + push_consumer.start() + while not stop_event.is_set(): + time.sleep(2) + if errors: + raise errors[0] diff --git a/tests/test_producer.py b/tests/test_producer.py new file mode 100644 index 0000000..311bf71 --- /dev/null +++ b/tests/test_producer.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from concurrent.futures import ThreadPoolExecutor +import time +import threading +import sys + +from rocketmq.client import Message, SendStatus, TransactionMQProducer, TransactionStatus + +PY_VERSION = sys.version_info[0] + + +def test_producer_send_sync(producer): + msg = Message('test') + msg.set_keys('send_sync') + msg.set_tags('XXX') + msg.set_body('XXXX') + ret = producer.send_sync(msg) + assert ret.status == SendStatus.OK + + +def test_producer_send_sync_multi_thread(producer): + executor = ThreadPoolExecutor(max_workers=5) + futures = [] + for _ in range(5): + futures.append(executor.submit(test_producer_send_sync, producer)) + + for future in futures: + _ret = future.result() + + +def test_producer_send_oneway(producer): + msg = Message('test') + msg.set_keys('send_oneway') + msg.set_tags('XXX') + msg.set_body('XXXX') + producer.send_oneway(msg) + + +def test_producer_send_orderly_with_sharding_key(orderly_producer): + msg = Message('test') + msg.set_keys('sharding_message') + msg.set_tags('sharding') + msg.set_body('sharding message') + msg.set_property('property', 'test') + ret = orderly_producer.send_orderly_with_sharding_key(msg, 'order1') + assert ret.status == SendStatus.OK + + +def test_transaction_producer(): + stop_event = threading.Event() + msg_body = 'XXXX' + + def on_local_execute(msg, user_args): + return TransactionStatus.UNKNOWN + + def on_check(msg): + stop_event.set() + assert msg.body.decode('utf-8') == msg_body + return TransactionStatus.COMMIT + + producer = TransactionMQProducer('transactionTestGroup' + str(PY_VERSION), on_check) + producer.set_name_server_address('127.0.0.1:9876') + producer.start() + msg = Message('test') + msg.set_keys('transaction') + msg.set_tags('XXX') + msg.set_body(msg_body) + producer.send_message_in_transaction(msg, on_local_execute) + while not stop_event.is_set(): + time.sleep(2) + producer.shutdown() + diff --git a/unitests/CMakeLists.txt b/unitests/CMakeLists.txt deleted file mode 100644 index 3e3e349..0000000 --- a/unitests/CMakeLists.txt +++ /dev/null @@ -1,73 +0,0 @@ -#/* -#* Licensed to the Apache Software Foundation (ASF) under one or more -#* contributor license agreements. See the NOTICE file distributed with -#* this work for additional information regarding copyright ownership. -#* The ASF licenses this file to You under the Apache License, Version 2.0 -#* (the "License"); you may not use this file except in compliance with -#* the License. You may obtain a copy of the License at -#* -#* http://www.apache.org/licenses/LICENSE-2.0 -#* -#* Unless required by applicable law or agreed to in writing, software -#* distributed under the License is distributed on an "AS IS" BASIS, -#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#* See the License for the specific language governing permissions and -#* limitations under the License. -#*/ -project(UnitTest) -if (WIN32) - find_package(gtest) -elseif (APPLE) - find_package(gtest) -else () - set(GTEST_INCLUDE_DIRS /usr/local/include) - set(GTEST_LIBRARY_DIRS /usr/local/lib) - set(GTEST_LIBRARIES /usr/local/lib/libgtest.a) - set(GTEST_FOUND true) -endif (WIN32) - -if (GTEST_FOUND) - message(STATUS "** GTEST Include dir: ${GTEST_INCLUDE_DIRS}") - message(STATUS "** GTEST Libraries dir: ${GTEST_LIBRARY_DIRS}") - message(STATUS "** GTEST Libraries: ${GTEST_LIBRARIES}") - include_directories(${GTEST_INCLUDE_DIRS}) -else () - message(FATAL_ERROR "Missing library: gtest ") -endif () -file(GLOB_RECURSE SRC_FILES ${CMAKE_SOURCE_DIR}/src/*.cpp) -file(GLOB_RECURSE TEST_SRC_FILES ${CMAKE_SOURCE_DIR}/unitests/*.cpp) - -# subdirs -SET(SUB_DIRS) -file(GLOB children ${CMAKE_SOURCE_DIR}/src/*.h) -FOREACH (child ${children}) - IF (IS_DIRECTORY ${child}) - LIST(APPEND SUB_DIRS ${child}) - ENDIF () -ENDFOREACH () -LIST(APPEND SUB_DIRS ${CMAKE_SOURCE_DIR}/src) -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${SUB_DIRS}) - -message(STATUS "All Source File" ${SRC_FILES}) -message(STATUS "All Test Source File" ${TEST_SRC_FILES}) -#message(STATUS "All Head File Dir" ${SUB_DIRS}) -add_executable(runUnitTests - ${TEST_SRC_FILES} - ${SRC_FILES} - ) - -target_link_libraries(runUnitTests - dl - ${ROCKETMQ_LIBRARIES} - ${Boost_LIBRARIES} - ${PYTHON_LIBRARIES} - ${GTEST_LIBRARIES}) - -if (UNIX AND NOT APPLE) - target_link_libraries(runUnitTests rt) -endif () - -set_target_properties(runUnitTests PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(runUnitTests ${CMAKE_BINARY_DIR}/runUnitTests) diff --git a/unitests/PythonWrapperTest.cpp b/unitests/PythonWrapperTest.cpp deleted file mode 100644 index 1004597..0000000 --- a/unitests/PythonWrapperTest.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gtest/gtest.h" -#include "PythonWrapper.h" - -int main(int argc, char **argv) { - testing::InitGoogleTest(&argc, argv); - int ret = RUN_ALL_TESTS(); - return ret; -} -TEST(Version, testVersion) { - const char *version = PyGetVersion(); - const char *versionExp = PyGetVersion(); - ASSERT_STREQ(version,versionExp); -} -TEST(Message, testCreateMessage) { - ASSERT_TRUE(1 == 1); -} - -TEST(Message, testSetMessageTopic) { - ASSERT_TRUE(1 == 1); -} - -TEST(Message, testSetMessageKey) { - ASSERT_TRUE(1 == 1); -} - -TEST(Message, testSetMessageTag) { - ASSERT_TRUE(1 == 1); -} - -TEST(Message, testSetMessageValue) { - ASSERT_TRUE(1 == 1); -} - -TEST(Message, testSetMessageDelayLevel) { - ASSERT_TRUE(1 == 1); -} - -TEST(Message, testDestroyMessage) { - ASSERT_TRUE(1 == 1); -}