New pubsub eventbus impl (#131)

* New pubsub eventbus impl

---------

Co-authored-by: Samir Ketema <samir@meroxa.io>

.github/workflows/build.yml

@@ -17,4 +17,4 @@ jobs:
           go-version-file: 'go.mod'
 
       - name: Test
-        run: make test GOTEST_FLAGS="-v -count=1"
+        run: make test

.gitignore

@@ -132,3 +132,5 @@ $RECYCLE.BIN/
 ### Project ###
 # Binary built with `make build`
 /conduit-connector-salesforce
+
+.env

.golangci.goheader.template

@@ -1,4 +1,4 @@
-Copyright © {{ copyright-year }} Meroxa, Inc. and Miquido
+Copyright © {{ copyright-year }} Meroxa, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

.golangci.yml

@@ -11,11 +11,12 @@ linters-settings:
     min-complexity: 20
   goconst:
     ignore-tests: true
-  goheader:
-    template-path: '.golangci.goheader.template'
-    values:
-      regexp:
-        copyright-year: 20[2-9]\d
+  # TODO: Skip this until ready to merge.
+  # goheader:
+  #  template-path: '.golangci.goheader.template'
+  #  values:
+  #    regexp:
+  #      copyright-year: 20[2-9]\d
 
 linters:
   # please, do not use `enable-all`: it's deprecated and will be removed soon.

.mockery.yaml

@@ -0,0 +1,24 @@
+---
+with-expecter: true
+inpackage: true
+disable-version-string: true
+filename: "mock_{{.InterfaceName | snakecase}}_test.go"
+mockname: "mock{{.InterfaceName | firstUpper}}"
+dir: "{{.InterfaceDir}}"
+packages:
+  github.com/conduitio-labs/conduit-connector-salesforce/proto/eventbus/v1:
+    interfaces:
+      PubSubClient:
+        config:
+          dir: source_pubsub
+          inpackage: false
+          outpkg: source
+      PubSub_SubscribeClient:
+        config:
+          dir: source_pubsub
+          inpackage: false
+          outpkg: source
+  github.com/conduitio-labs/conduit-connector-salesforce/source:
+    interfaces:
+      authenticator: null
+      client: null

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build test lint
+.PHONY: build test lint fmt proto
 
 VERSION=$(shell git describe --tags --dirty --always)
 
@@ -8,8 +8,20 @@ build:
 test:
 	go test $(GOTEST_FLAGS) -race ./...
 
+fmt:
+	gofumpt -l -w .
+	gci write --skip-generated  .
+
 lint:
-	golangci-lint run
+	golangci-lint run -v
+
+proto:
+	cd proto && buf generate
+
+generate:
+	go generate ./...
+	mockery
+
 
 .PHONY: install-tools
 install-tools:

README.md

@@ -11,7 +11,7 @@ Run `make`.
 
 ## Source
 
-The Source connector subscribes to given list of topics and listens for events published by Salesforce.
+The Source connector subscribes to Salesforce platform events and queries published events in real time.
 
 ### How it works
 
@@ -60,6 +60,8 @@ There may be need to reconnect while connector is working. Server returns condit
 
 ### Configuration Options
 
+## Destination
+
 | name              | description                                                                                                                                                                                                                                    | required | default |
 |-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------|
 | `environment`     | Authorization service based on Organization’s Domain Name (e.g.: https://MyDomainName.my.salesforce.com -> `MyDomainName`) or `sandbox` for test environment.                                                                                  | `true`   |         |
@@ -71,9 +73,28 @@ There may be need to reconnect while connector is working. Server returns condit
 | `pushTopicsNames` | The comma-separated list of names of the Push Topics to listen to. All values will be prefixed with `/topic/`. All values will have white spaces trimmed. Any empty value is skipped. All Topics have to exist for connector to start working. | `true`   |         |
 | `keyField`        | The name of the Response's field that should be used as a Payload's Key. Empty value will set it to `nil`.                                                                                                                                     | `false`  | `"Id"`  |
 
+## Source
+
+
+| name              | description                                                                                                                                                                                                                                    | required | default |
+|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------|
+| `oauthEndpoint`     | Authorization service based on Organization’s Domain Name (e.g.: https://MyDomainName.my.salesforce.com ) |   | `true`  |
+| `clientId`        | OAuth Client ID (Consumer Key)           | `true`   |         |
+| `clientSecret`    | OAuth Client Secret (Consumer Secret)      | `true`   |         |
+| `username`        | Username.  | `false`   |         |
+| ~~`topicName`~~ | Event topic name for your event (e.g: /event/Accepted_Quote__e) **Deprecated: use `topicNames` instead**  |	`false` ||
+| `topicsNames`        | One or multiple comma separated topic names the source will subscribe to (e.g */event/Test__e, /event/Test2__e*).  | `true`   ||
+| `retryCount`        | Number of times the connector will retry is the connection to a topic breaks.  | `false`   |    `10`     |
+| `replayPreset`        | The position from which connector will start reading events, either 'latest' or 'earliest'. Latest will pull only newly created events, and earlies will pull any events that are currently in the topic.  | `false`   |    `earliest`     |
+| `pollingPeriod`        | The client event polling interval between each data read on the topic. | `false`   |    `100ms`     |
+| `insecureSkipVerify`        | Disables certificate validation. | `false`   |   `false`      |
+
+
 ### Step-by-step configuration example
 
-There are a couple of steps that need to be done to start working with Salesforce connector.
+## Destination
+
+There are a couple of steps that need to be done to start working with Salesforce connector as destination.
 
 1. Log in into Your Salesforce account, e.g. https://my-demo-app.my.salesforce.com. The environment is `my-demo-app`.
 2. First, if not already done, You need to create connected app and enable OAuth: [Enable OAuth Settings for API Integration](https://help.salesforce.com/s/articleView?id=sf.connected_app_create_api_integration.htm&type=5).
@@ -105,6 +126,32 @@ There are a couple of steps that need to be done to start working with Salesforc
 
         Later, this may be utilized by other connectors, e.g. [Elasticsearch connector](https://github.com/conduitio-labs/conduit-connector-elasticsearch) will create Document with ID of Record's Key when available.
 
+## Source
+
+1. Log in into Your Salesforce account, e.g. https://my-demo-app.my.salesforce.com. The environment is `my-demo-app`.
+2. First, if not already done, You need to create connected app and enable OAuth: [Enable OAuth Settings for API Integration](https://help.salesforce.com/s/articleView?id=sf.connected_app_create_api_integration.htm&type=5).
+
+    The callback URL is required, but not relevant for this connector, so you can put anything there.
+
+    Successfully configured app example can be seen below:
+
+    ![Connected App example](docs/connect_and_configure_app.png)
+3. Copy **Consumer Key** and **Consumer Secret**. If You need these values once again You can always find them in _Setup -> Apps -> App Manager_, find app on the list and choose _View_ option.
+    ![View OAuth tokens](docs/view_oauth_tokens.png)
+
+4. You will need to set following settings on the application, refer to ![OAuth Client Credentials Flow Requirements](https://help.salesforce.com/s/articleView?id=sf.connected_app_client_credentials_setup.htm&type=5) .
+    Make sure the user you are attaching to the application has "API ENABLED" permission set on their account.
+
+5. Set up event:
+
+6. Once done, You can begin with configuring the connector:
+    1. Set `oauthEndpount`, it should be a full url to your salesforce, (e.g: https://mysalesforce.salesforce.com )
+    2. Use Step 3 **Consumer Key** value as `clientId` config.
+    3. Use Step 3 **Consumer Secret** value as `clientSecret` config.
+    4. Set `topicNames` to either one or multiple event topic names the connector will subscribe to, should contain event in topic name such as `/event/Test__e`.
+    5. Set any optional config fields as needed.
+
+
 ## Testing
 
 Run `make test` to run all the unit and integration tests.

cmd/connector/acceptance_test.go

@@ -1,4 +1,4 @@
-// Copyright © 2022 Meroxa, Inc. and Miquido
+// Copyright © 2022 Meroxa, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

cmd/connector/main.go

@@ -1,4 +1,4 @@
-// Copyright © 2022 Meroxa, Inc. and Miquido
+// Copyright © 2022 Meroxa, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,14 +16,14 @@ package main
 
 import (
 	sf "github.com/conduitio-labs/conduit-connector-salesforce"
-	sfSource "github.com/conduitio-labs/conduit-connector-salesforce/source"
+	sfSourcePubsub "github.com/conduitio-labs/conduit-connector-salesforce/source_pubsub"
 	sdk "github.com/conduitio/conduit-connector-sdk"
 )
 
 func main() {
 	sdk.Serve(sdk.Connector{
 		NewSpecification: sf.Specification,
-		NewSource:        sfSource.NewSource,
+		NewSource:        sfSourcePubsub.NewSource,
 		NewDestination:   nil,
 	})
 }

cmd/connector/oauth_client_mock_test.go

@@ -1,4 +1,4 @@
-// Copyright © 2022 Meroxa, Inc. and Miquido
+// Copyright © 2022 Meroxa, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,8 +20,7 @@ import (
 	"github.com/conduitio-labs/conduit-connector-salesforce/internal/salesforce/oauth/response"
 )
 
-type oAuthClientMock struct {
-}
+type oAuthClientMock struct{}
 
 func (c *oAuthClientMock) Authenticate(_ context.Context) (response.TokenResponse, error) {
 	return response.TokenResponse{

cmd/connector/streaming_client_mock_test.go

@@ -1,4 +1,4 @@
-// Copyright © 2022 Meroxa, Inc. and Miquido
+// Copyright © 2022 Meroxa, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.