From e02e15e50a184040ed4c4dce18e627546f2b1b06 Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Sat, 13 Jul 2024 13:35:04 -0400 Subject: [PATCH] autoformat cpp code Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- .../adapters/kafka/KafkaAdapterManager.cpp | 140 +-- cpp/csp/adapters/kafka/KafkaConsumer.cpp | 199 +++-- cpp/csp/adapters/kafka/KafkaInputAdapter.cpp | 109 ++- cpp/csp/adapters/kafka/KafkaOutputAdapter.cpp | 63 +- cpp/csp/adapters/kafka/KafkaPublisher.cpp | 63 +- cpp/csp/adapters/kafka/KafkaSubscriber.cpp | 28 +- .../parquet/ArrowIPCFileReaderWrapper.cpp | 39 +- .../parquet/ArrowIPCFileWriterWrapper.cpp | 27 +- .../adapters/parquet/FileReaderWrapper.cpp | 16 +- .../adapters/parquet/FileWriterWrapper.cpp | 38 +- .../parquet/FileWriterWrapperContainer.cpp | 91 +- .../parquet/ParquetDictBasketOutputWriter.cpp | 114 ++- .../parquet/ParquetFileReaderWrapper.cpp | 28 +- .../parquet/ParquetFileWriterWrapper.cpp | 37 +- .../parquet/ParquetInputAdapterManager.cpp | 324 +++---- .../adapters/parquet/ParquetOutputAdapter.cpp | 252 +++--- .../parquet/ParquetOutputAdapterManager.cpp | 89 +- .../parquet/ParquetOutputFilenameAdapter.cpp | 6 +- cpp/csp/adapters/parquet/ParquetReader.cpp | 323 ++++--- .../parquet/ParquetReaderColumnAdapter.cpp | 778 ++++++++--------- cpp/csp/adapters/parquet/ParquetWriter.cpp | 178 ++-- .../utils/JSONMessageStructConverter.cpp | 158 ++-- cpp/csp/adapters/utils/MessageEnums.cpp | 18 +- .../adapters/utils/MessageStructConverter.cpp | 32 +- cpp/csp/adapters/utils/MessageWriter.cpp | 79 +- cpp/csp/adapters/utils/ProtobufHelper.cpp | 365 +++++--- .../utils/ProtobufMessageStructConverter.cpp | 32 +- .../utils/RawBytesMessageStructConverter.cpp | 36 +- .../websocket/ClientAdapterManager.cpp | 146 ++-- .../websocket/ClientHeaderUpdateAdapter.cpp | 22 +- .../adapters/websocket/ClientInputAdapter.cpp | 32 +- .../websocket/ClientOutputAdapter.cpp | 15 +- .../adapters/websocket/WebsocketEndpoint.cpp | 76 +- cpp/csp/core/Exception.cpp | 132 ++- cpp/csp/core/Time.cpp | 53 +- cpp/csp/cppnodes/baselibimpl.cpp | 332 ++++--- cpp/csp/cppnodes/basketlibimpl.cpp | 32 +- cpp/csp/cppnodes/mathimpl.cpp | 376 +++++--- cpp/csp/cppnodes/statsimpl.cpp | 178 ++-- cpp/csp/engine/AdapterManager.cpp | 36 +- cpp/csp/engine/BasketInfo.cpp | 180 ++-- cpp/csp/engine/Consumer.cpp | 23 +- cpp/csp/engine/CspEnum.cpp | 22 +- cpp/csp/engine/CspType.cpp | 35 +- cpp/csp/engine/CycleStepTable.cpp | 48 +- cpp/csp/engine/Dictionary.cpp | 34 +- cpp/csp/engine/DynamicEngine.cpp | 33 +- cpp/csp/engine/DynamicNode.cpp | 174 ++-- cpp/csp/engine/Engine.cpp | 147 ++-- cpp/csp/engine/Enums.cpp | 7 +- cpp/csp/engine/EventPropagator.cpp | 53 +- cpp/csp/engine/GraphOutputAdapter.cpp | 42 +- cpp/csp/engine/InputAdapter.cpp | 9 +- cpp/csp/engine/Node.cpp | 58 +- cpp/csp/engine/OutputAdapter.cpp | 13 +- cpp/csp/engine/PendingPushEvents.cpp | 67 +- cpp/csp/engine/PushPullInputAdapter.cpp | 72 +- cpp/csp/engine/RootEngine.cpp | 109 ++- cpp/csp/engine/Scheduler.cpp | 211 +++-- cpp/csp/engine/StatusAdapter.cpp | 8 +- cpp/csp/engine/Struct.cpp | 425 +++++---- cpp/csp/engine/TimeSeries.cpp | 14 +- cpp/csp/engine/TimeSeriesProvider.cpp | 23 +- cpp/csp/engine/VectorContainer.cpp | 42 +- cpp/csp/python/Conversions.cpp | 18 +- cpp/csp/python/CspTypeFactory.cpp | 66 +- cpp/csp/python/NumpyConversions.cpp | 74 +- cpp/csp/python/PyAdapterManager.cpp | 148 ++-- cpp/csp/python/PyAdapterManagerWrapper.cpp | 15 +- cpp/csp/python/PyBasketInputProxy.cpp | 717 ++++++++-------- cpp/csp/python/PyBasketOutputProxy.cpp | 516 +++++------ cpp/csp/python/PyConstAdapter.cpp | 33 +- cpp/csp/python/PyConstants.cpp | 60 +- cpp/csp/python/PyCppNode.cpp | 32 +- cpp/csp/python/PyCspEnum.cpp | 287 +++---- cpp/csp/python/PyCspType.cpp | 44 +- cpp/csp/python/PyDynamicNode.cpp | 169 ++-- cpp/csp/python/PyEngine.cpp | 157 ++-- cpp/csp/python/PyFeedbackAdapter.cpp | 26 +- cpp/csp/python/PyGraphOutputAdapter.cpp | 66 +- cpp/csp/python/PyInputAdapterWrapper.cpp | 65 +- cpp/csp/python/PyInputProxy.cpp | 512 +++++------ cpp/csp/python/PyManagedSimInputAdapter.cpp | 175 ++-- cpp/csp/python/PyNode.cpp | 202 ++--- cpp/csp/python/PyNodeWrapper.cpp | 201 +++-- cpp/csp/python/PyNumbaNode.cpp | 272 +++--- cpp/csp/python/PyNumpyAdapter.cpp | 26 +- cpp/csp/python/PyOutputAdapter.cpp | 54 +- cpp/csp/python/PyOutputAdapterWrapper.cpp | 136 ++- cpp/csp/python/PyOutputProxy.cpp | 153 ++-- cpp/csp/python/PyPullInputAdapter.cpp | 59 +- cpp/csp/python/PyPushInputAdapter.cpp | 273 +++--- cpp/csp/python/PyPushPullInputAdapter.cpp | 178 ++-- cpp/csp/python/PyStruct.cpp | 809 ++++++++++-------- cpp/csp/python/PyStructToDict.cpp | 103 +-- cpp/csp/python/PyStructToJson.cpp | 168 ++-- cpp/csp/python/PyTimerAdapter.cpp | 23 +- cpp/csp/python/adapters/kafkaadapterimpl.cpp | 51 +- .../python/adapters/parquetadapterimpl.cpp | 616 ++++++------- .../python/adapters/websocketadapterimpl.cpp | 63 +- cpp/csp/python/cspbaselibimpl.cpp | 187 ++-- cpp/csp/python/cspbasketlibimpl.cpp | 15 +- cpp/csp/python/cspimpl.cpp | 94 +- cpp/csp/python/cspmathimpl.cpp | 17 +- cpp/csp/python/cspnpstatsimpl.cpp | 100 +-- cpp/csp/python/cspstatsimpl.cpp | 87 +- cpp/csp/python/csptestlibimpl.cpp | 132 ++- cpp/csp/python/csptypesimpl.cpp | 17 +- cpp/csp/python/npstatsimpl.cpp | 624 +++++++------- cpp/tests/core/test_basic_allocator.cpp | 42 +- cpp/tests/core/test_dynamicbitset.cpp | 30 +- cpp/tests/core/test_enum.cpp | 21 +- cpp/tests/core/test_platform.cpp | 10 +- cpp/tests/core/test_srmwlockfreequeue.cpp | 80 +- cpp/tests/core/test_tagged_pointer_union.cpp | 8 +- cpp/tests/core/test_time.cpp | 116 +-- cpp/tests/engine/test_dictionary.cpp | 75 +- cpp/tests/engine/test_engine_initial.cpp | 80 +- .../engine/test_partial_switch_csp_type.cpp | 313 +++---- cpp/tests/engine/test_tick_buffer.cpp | 24 +- cpp/tests/engine/test_time_series.cpp | 33 +- cpp/tests/engine/test_window_buffer.cpp | 47 +- 122 files changed, 7883 insertions(+), 7777 deletions(-) diff --git a/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp b/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp index 06d5fce9..ad8b4cdd 100644 --- a/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp +++ b/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp @@ -2,8 +2,8 @@ #include #include #include -#include #include +#include #include #include @@ -11,12 +11,8 @@ namespace csp { -INIT_CSP_ENUM( csp::adapters::kafka::KafkaStatusMessageType, - "OK", - "MSG_DELIVERY_FAILED", - "MSG_SEND_ERROR", - "MSG_RECV_ERROR" -); +INIT_CSP_ENUM( csp::adapters::kafka::KafkaStatusMessageType, "OK", "MSG_DELIVERY_FAILED", "MSG_SEND_ERROR", + "MSG_RECV_ERROR" ); } @@ -26,20 +22,23 @@ namespace csp::adapters::kafka class DeliveryReportCb : public RdKafka::DeliveryReportCb { public: - DeliveryReportCb( KafkaAdapterManager * mgr ) : m_adapterManager( mgr ) + DeliveryReportCb( KafkaAdapterManager * mgr ) + : m_adapterManager( mgr ) { } - void dr_cb( RdKafka::Message &message ) final + void dr_cb( RdKafka::Message & message ) final { /* If message.err() is non-zero the message delivery failed permanently * for the message. */ if( message.err() ) { - std::string msg = "KafkaPublisher: Message delivery failed for topic " + message.topic_name() + ". Failure: " + message.errstr(); - m_adapterManager -> pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_DELIVERY_FAILED, msg ); + std::string msg = "KafkaPublisher: Message delivery failed for topic " + message.topic_name() + + ". Failure: " + message.errstr(); + m_adapterManager->pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_DELIVERY_FAILED, msg ); } } + private: KafkaAdapterManager * m_adapterManager; }; @@ -47,7 +46,10 @@ class DeliveryReportCb : public RdKafka::DeliveryReportCb class EventCb : public RdKafka::EventCb { public: - EventCb( KafkaAdapterManager * mgr ) : m_adapterManager( mgr ) {} + EventCb( KafkaAdapterManager * mgr ) + : m_adapterManager( mgr ) + { + } void event_cb( RdKafka::Event & event ) override { @@ -55,15 +57,17 @@ class EventCb : public RdKafka::EventCb { if( event.severity() < RdKafka::Event::EVENT_SEVERITY_NOTICE ) { - std::string errmsg = "KafkaConsumer: error " + RdKafka::err2str( ( RdKafka::ErrorCode ) event.err() ) + ". Reason: " + event.str(); - m_adapterManager -> pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::GENERIC_ERROR, errmsg ); + std::string errmsg = "KafkaConsumer: error " + RdKafka::err2str( (RdKafka::ErrorCode)event.err() ) + + ". Reason: " + event.str(); + m_adapterManager->pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::GENERIC_ERROR, errmsg ); } } else if( event.type() == RdKafka::Event::EVENT_ERROR ) { - //We shutdown the app if its a fatal error OR if its an authentication issue which has plagued users multiple times + // We shutdown the app if its a fatal error OR if its an authentication issue which has plagued users + // multiple times if( event.fatal() || event.err() == RdKafka::ErrorCode::ERR__AUTHENTICATION ) - m_adapterManager -> forceShutdown( RdKafka::err2str( ( RdKafka::ErrorCode ) event.err() ) + event.str() ); + m_adapterManager->forceShutdown( RdKafka::err2str( (RdKafka::ErrorCode)event.err() ) + event.str() ); } } @@ -71,14 +75,15 @@ class EventCb : public RdKafka::EventCb KafkaAdapterManager * m_adapterManager; }; -KafkaAdapterManager::KafkaAdapterManager( csp::Engine * engine, const Dictionary & properties ) : AdapterManager( engine ), - m_consumerIdx( 0 ), - m_producerPollThreadActive( false ) +KafkaAdapterManager::KafkaAdapterManager( csp::Engine * engine, const Dictionary & properties ) + : AdapterManager( engine ) + , m_consumerIdx( 0 ) + , m_producerPollThreadActive( false ) { - m_maxThreads = properties.get( "max_threads" ); + m_maxThreads = properties.get( "max_threads" ); m_pollTimeoutMs = properties.get( "poll_timeout" ).asMilliseconds(); - m_eventCb = std::make_unique( this ); + m_eventCb = std::make_unique( this ); m_producerCb = std::make_unique( this ); std::string errstr; @@ -90,19 +95,19 @@ KafkaAdapterManager::KafkaAdapterManager( csp::Engine * engine, const Dictionary setConfProperties( m_consumerConf.get(), *properties.get( "rd_kafka_consumer_conf_properties" ) ); if( properties.exists( "start_offset" ) ) { - //used later in start since we need starttime + // used later in start since we need starttime m_startOffsetProperty = properties.getUntypedValue( "start_offset" ); } - if( m_consumerConf -> set( "event_cb", m_eventCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) + if( m_consumerConf->set( "event_cb", m_eventCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) CSP_THROW( RuntimeException, "Failed to set consumer error cb: " << errstr ); m_producerConf.reset( RdKafka::Conf::create( RdKafka::Conf::CONF_GLOBAL ) ); setConfProperties( m_producerConf.get(), rdKafkaProperties ); setConfProperties( m_producerConf.get(), *properties.get( "rd_kafka_producer_conf_properties" ) ); - if( m_producerConf -> set( "dr_cb", m_producerCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) + if( m_producerConf->set( "dr_cb", m_producerCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) CSP_THROW( RuntimeException, "Failed to set producer callback: " << errstr ); - if( m_producerConf -> set( "event_cb", m_eventCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) + if( m_producerConf->set( "event_cb", m_eventCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) CSP_THROW( RuntimeException, "Failed to set producer error cb: " << errstr ); } @@ -112,7 +117,7 @@ KafkaAdapterManager::~KafkaAdapterManager() if( m_producerPollThreadActive ) { m_producerPollThreadActive = false; - m_producerPollThread -> join(); + m_producerPollThread->join(); } } @@ -122,9 +127,9 @@ void KafkaAdapterManager::setConfProperties( RdKafka::Conf * conf, const Diction for( auto it = properties.begin(); it != properties.end(); ++it ) { - std::string key = it.key(); + std::string key = it.key(); std::string value = properties.get( key ); - if( conf -> set( key, value, errstr ) != RdKafka::Conf::CONF_OK ) + if( conf->set( key, value, errstr ) != RdKafka::Conf::CONF_OK ) CSP_THROW( RuntimeException, "Failed to set property " << key << ": " << errstr ); } } @@ -134,18 +139,18 @@ void KafkaAdapterManager::forceShutdown( const std::string & err ) forceConsumerReplayComplete(); try { - CSP_THROW( RuntimeException, "Kafka fatal error. " + err ); + CSP_THROW( RuntimeException, "Kafka fatal error. " + err ); } catch( const RuntimeException & ) { - rootEngine() -> shutdown( std::current_exception() ); + rootEngine()->shutdown( std::current_exception() ); } } void KafkaAdapterManager::forceConsumerReplayComplete() { for( auto & consumer : m_consumerVector ) - consumer -> forceReplayCompleted(); + consumer->forceReplayCompleted(); } void KafkaAdapterManager::start( DateTime starttime, DateTime endtime ) @@ -155,7 +160,7 @@ void KafkaAdapterManager::start( DateTime starttime, DateTime endtime ) if( !m_staticPublishers.empty() || !m_dynamicPublishers.empty() ) { m_producer.reset( RdKafka::Producer::create( m_producerConf.get(), errstr ) ); - if ( !m_producer ) + if( !m_producer ) { CSP_THROW( RuntimeException, "Failed to create producer: " << errstr ); } @@ -163,21 +168,21 @@ void KafkaAdapterManager::start( DateTime starttime, DateTime endtime ) // start all consumers for( auto & it : m_consumerVector ) - it -> start( starttime ); + it->start( starttime ); // start all publishers for( auto & it : m_staticPublishers ) - it.second -> start( m_producer ); + it.second->start( m_producer ); for( auto & it : m_dynamicPublishers ) - it -> start( m_producer ); + it->start( m_producer ); AdapterManager::start( starttime, endtime ); if( !m_staticPublishers.empty() || !m_dynamicPublishers.empty() ) { m_producerPollThreadActive = true; - m_producerPollThread = std::make_unique( [ this ](){ pollProducers(); } ); + m_producerPollThread = std::make_unique( [this]() { pollProducers(); } ); } } @@ -187,20 +192,20 @@ void KafkaAdapterManager::stop() // stop all consumers for( auto & it : m_consumerVector ) - it -> stop(); + it->stop(); if( m_producerPollThreadActive ) { m_producerPollThreadActive = false; - m_producerPollThread -> join(); + m_producerPollThread->join(); } // stop all publishers for( auto & it : m_staticPublishers ) - it.second -> stop(); + it.second->stop(); for( auto & it : m_dynamicPublishers ) - it -> stop(); + it->stop(); m_staticPublishers.clear(); m_dynamicPublishers.clear(); @@ -218,33 +223,35 @@ void KafkaAdapterManager::pollProducers() { while( m_producerPollThreadActive ) { - m_producer -> poll( 1000 ); + m_producer->poll( 1000 ); } try { while( true ) { - auto rc = m_producer -> flush( 10000 ); + auto rc = m_producer->flush( 10000 ); if( !rc ) break; if( rc && rc != RdKafka::ERR__TIMED_OUT ) - CSP_THROW( RuntimeException, "KafkaProducer failed to flush pending msgs on shutdown: " << RdKafka::err2str( rc ) ); + CSP_THROW( RuntimeException, + "KafkaProducer failed to flush pending msgs on shutdown: " << RdKafka::err2str( rc ) ); } } catch( ... ) { - rootEngine() -> shutdown( std::current_exception() ); + rootEngine()->shutdown( std::current_exception() ); } } -PushInputAdapter * KafkaAdapterManager::getInputAdapter( CspTypePtr & type, PushMode pushMode, const Dictionary & properties ) +PushInputAdapter * KafkaAdapterManager::getInputAdapter( CspTypePtr & type, PushMode pushMode, + const Dictionary & properties ) { - std::string topic = properties.get( "topic" ); - std::string key = properties.get( "key" ); - KafkaSubscriber * subscriber = this -> getSubscriber( topic, key, properties ); - return subscriber -> getInputAdapter( type, pushMode, properties ); + std::string topic = properties.get( "topic" ); + std::string key = properties.get( "key" ); + KafkaSubscriber * subscriber = this->getSubscriber( topic, key, properties ); + return subscriber->getInputAdapter( type, pushMode, properties ); } OutputAdapter * KafkaAdapterManager::getOutputAdapter( CspTypePtr & type, const Dictionary & properties ) @@ -252,10 +259,10 @@ OutputAdapter * KafkaAdapterManager::getOutputAdapter( CspTypePtr & type, const std::string topic = properties.get( "topic" ); try { - auto key = properties.get( "key" ); - auto pair = TopicKeyPair( topic, key ); - KafkaPublisher * publisher = this -> getStaticPublisher( pair, properties ); - return publisher -> getOutputAdapter( type, properties, key ); + auto key = properties.get( "key" ); + auto pair = TopicKeyPair( topic, key ); + KafkaPublisher * publisher = this->getStaticPublisher( pair, properties ); + return publisher->getOutputAdapter( type, properties, key ); } catch( TypeError & e ) { @@ -264,8 +271,8 @@ OutputAdapter * KafkaAdapterManager::getOutputAdapter( CspTypePtr & type, const for( auto & it : key ) keyFields.emplace_back( std::get( it._data ) ); - KafkaPublisher * publisher = this -> getDynamicPublisher( topic, properties ); - return publisher -> getOutputAdapter( type, properties, keyFields ); + KafkaPublisher * publisher = this->getDynamicPublisher( topic, properties ); + return publisher->getOutputAdapter( type, properties, keyFields ); } } @@ -276,37 +283,38 @@ KafkaConsumer * KafkaAdapterManager::getConsumer( const std::string & topic, con // If we have reached m_maxThreads, then round-robin the topic onto a consumer (and insert it into the map) if( m_consumerMap.find( topic ) != m_consumerMap.end() ) { - return m_consumerMap[ topic ].get(); + return m_consumerMap[topic].get(); } if( m_consumerVector.size() < m_maxThreads ) { auto consumer = std::make_shared( this, properties ); m_consumerVector.emplace_back( consumer ); m_consumerMap.emplace( topic, consumer ); - return m_consumerMap[ topic ].get(); + return m_consumerMap[topic].get(); } - auto consumer = m_consumerVector[ m_consumerIdx++ ]; + auto consumer = m_consumerVector[m_consumerIdx++]; m_consumerMap.emplace( topic, consumer ); if( m_consumerIdx >= m_maxThreads ) m_consumerIdx = 0; return consumer.get(); } -KafkaSubscriber * KafkaAdapterManager::getSubscriber( const std::string & topic, const std::string & key, const Dictionary & properties ) +KafkaSubscriber * KafkaAdapterManager::getSubscriber( const std::string & topic, const std::string & key, + const Dictionary & properties ) { auto pair = TopicKeyPair( topic, key ); - auto rv = m_subscribers.emplace( pair, nullptr ); + auto rv = m_subscribers.emplace( pair, nullptr ); if( rv.second ) { std::unique_ptr subscriber( new KafkaSubscriber( this, properties ) ); - rv.first -> second = std::move( subscriber ); + rv.first->second = std::move( subscriber ); - this -> getConsumer( topic, properties ) -> addSubscriber( topic, key, rv.first -> second.get() ); + this->getConsumer( topic, properties )->addSubscriber( topic, key, rv.first->second.get() ); } - return rv.first -> second.get(); + return rv.first->second.get(); } // for static (string) keys, we create one publisher instance per pair @@ -317,10 +325,10 @@ KafkaPublisher * KafkaAdapterManager::getStaticPublisher( const TopicKeyPair & p if( rv.second ) { std::unique_ptr publisher( new KafkaPublisher( this, properties, pair.first ) ); - rv.first -> second = std::move( publisher ); + rv.first->second = std::move( publisher ); } - KafkaPublisher * p = rv.first -> second.get(); + KafkaPublisher * p = rv.first->second.get(); return p; } @@ -332,4 +340,4 @@ KafkaPublisher * KafkaAdapterManager::getDynamicPublisher( const std::string & t return p; } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaConsumer.cpp b/cpp/csp/adapters/kafka/KafkaConsumer.cpp index 3f3d7dac..5cdc367b 100644 --- a/cpp/csp/adapters/kafka/KafkaConsumer.cpp +++ b/cpp/csp/adapters/kafka/KafkaConsumer.cpp @@ -8,26 +8,26 @@ namespace csp::adapters::kafka class RebalanceCb : public RdKafka::RebalanceCb { public: - RebalanceCb( KafkaConsumer & consumer ) : m_consumer( consumer ), - m_startOffset( RdKafka::Topic::OFFSET_INVALID ), - m_doneSeeking( false ) + RebalanceCb( KafkaConsumer & consumer ) + : m_consumer( consumer ) + , m_startOffset( RdKafka::Topic::OFFSET_INVALID ) + , m_doneSeeking( false ) { } void setStartOffset( int64_t offset ) { m_startOffset = offset; } - void setStartTime( DateTime time ) { m_startTime = time; } + void setStartTime( DateTime time ) { m_startTime = time; } - void rebalance_cb( RdKafka::KafkaConsumer *consumer, - RdKafka::ErrorCode err, - std::vector & partitions ) override + void rebalance_cb( RdKafka::KafkaConsumer * consumer, RdKafka::ErrorCode err, + std::vector & partitions ) override { if( err == RdKafka::ERR__ASSIGN_PARTITIONS ) { if( !m_doneSeeking ) { - std::unordered_map numPartitions; + std::unordered_map numPartitions; for( auto * partition : partitions ) - numPartitions[ partition -> topic() ] += 1; + numPartitions[partition->topic()] += 1; for( auto & entry : numPartitions ) m_consumer.setNumPartitions( entry.first, entry.second ); @@ -35,49 +35,54 @@ class RebalanceCb : public RdKafka::RebalanceCb if( !m_startTime.isNone() ) { for( auto * partition : partitions ) - partition -> set_offset( m_startTime.asMilliseconds() ); + partition->set_offset( m_startTime.asMilliseconds() ); - auto rc = consumer -> offsetsForTimes( partitions, 10000 ); + auto rc = consumer->offsetsForTimes( partitions, 10000 ); if( rc ) - CSP_THROW( RuntimeException, "Failed to get kafka offsets for starttime " << m_startTime << ": " << RdKafka::err2str( rc ) ); + CSP_THROW( RuntimeException, + "Failed to get kafka offsets for starttime " << m_startTime << ": " + << RdKafka::err2str( rc ) ); } else { for( auto * partition : partitions ) - partition -> set_offset( m_startOffset ); + partition->set_offset( m_startOffset ); } - auto rc = consumer -> assign( partitions ); + auto rc = consumer->assign( partitions ); if( rc ) - CSP_THROW( RuntimeException, "Failed to get kafka offsets for starttime " << m_startTime << ": " << RdKafka::err2str( rc ) ); + CSP_THROW( RuntimeException, + "Failed to get kafka offsets for starttime " << m_startTime << ": " + << RdKafka::err2str( rc ) ); m_doneSeeking = true; } else - consumer -> assign( partitions ); + consumer->assign( partitions ); } else { - consumer -> unassign(); + consumer->unassign(); } } private: KafkaConsumer & m_consumer; - DateTime m_startTime; - int64_t m_startOffset; - bool m_doneSeeking; + DateTime m_startTime; + int64_t m_startOffset; + bool m_doneSeeking; }; -KafkaConsumer::KafkaConsumer( KafkaAdapterManager * mgr, const Dictionary & properties ) : m_mgr( mgr ), - m_running( false ) +KafkaConsumer::KafkaConsumer( KafkaAdapterManager * mgr, const Dictionary & properties ) + : m_mgr( mgr ) + , m_running( false ) { - if( mgr -> startOffsetProperty().index() > 0 ) + if( mgr->startOffsetProperty().index() > 0 ) m_rebalanceCb = std::make_unique( *this ); std::string errstr; - auto * conf = m_mgr -> getConsumerConf(); - if( conf -> set( "rebalance_cb", m_rebalanceCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) + auto * conf = m_mgr->getConsumerConf(); + if( conf->set( "rebalance_cb", m_rebalanceCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) CSP_THROW( RuntimeException, "Failed to set rebalance callback: " << errstr ); m_consumer.reset( RdKafka::KafkaConsumer::create( conf, errstr ) ); @@ -101,38 +106,45 @@ void KafkaConsumer::addSubscriber( const std::string & topic, const std::string } else m_topics[topic].subscribers[key].emplace_back( subscriber ); - //This is a bit convoluted, but basically if we dont have rebalanceCB set, that means we are in "groupid" mode - //which doesnt support seeking. We force the adapters into a live mode, because groupid mode leads to deadlocks - //on adapters that dont received any data since we dont have partition information available to declare them done ( we dont even connect to them all ) + // This is a bit convoluted, but basically if we dont have rebalanceCB set, that means we are in "groupid" mode + // which doesnt support seeking. We force the adapters into a live mode, because groupid mode leads to deadlocks + // on adapters that dont received any data since we dont have partition information available to declare them done ( + // we dont even connect to them all ) if( !m_rebalanceCb ) - subscriber -> flagReplayComplete(); + subscriber->flagReplayComplete(); } void KafkaConsumer::start( DateTime starttime ) { - //RebalanceCb is only used / available if we requested a start_offset + // RebalanceCb is only used / available if we requested a start_offset if( m_rebalanceCb ) { - auto & startOffsetProperty = m_mgr -> startOffsetProperty(); + auto & startOffsetProperty = m_mgr->startOffsetProperty(); if( std::holds_alternative( startOffsetProperty ) ) { - KafkaStartOffset sOffset = ( KafkaStartOffset ) std::get( startOffsetProperty ); + KafkaStartOffset sOffset = (KafkaStartOffset)std::get( startOffsetProperty ); switch( sOffset ) { - case KafkaStartOffset::EARLIEST: m_rebalanceCb -> setStartOffset( RdKafka::Topic::OFFSET_BEGINNING ); break; - case KafkaStartOffset::LATEST: m_rebalanceCb -> setStartOffset( RdKafka::Topic::OFFSET_END ); break; - case KafkaStartOffset::START_TIME: m_rebalanceCb -> setStartTime( starttime ); break; + case KafkaStartOffset::EARLIEST: + m_rebalanceCb->setStartOffset( RdKafka::Topic::OFFSET_BEGINNING ); + break; + case KafkaStartOffset::LATEST: + m_rebalanceCb->setStartOffset( RdKafka::Topic::OFFSET_END ); + break; + case KafkaStartOffset::START_TIME: + m_rebalanceCb->setStartTime( starttime ); + break; } } else if( std::holds_alternative( startOffsetProperty ) ) { auto dt = std::get( startOffsetProperty ); - m_rebalanceCb -> setStartTime( dt ); + m_rebalanceCb->setStartTime( dt ); } else if( std::holds_alternative( startOffsetProperty ) ) { auto delta = std::get( startOffsetProperty ); - m_rebalanceCb -> setStartTime( starttime - delta.abs() ); + m_rebalanceCb->setStartTime( starttime - delta.abs() ); } else CSP_THROW( TypeError, "Expected enum, datetime or timedelta for startOffset" ); @@ -144,12 +156,13 @@ void KafkaConsumer::start( DateTime starttime ) for( auto & entry : m_topics ) topics.emplace_back( entry.first ); - RdKafka::ErrorCode err = m_consumer -> subscribe( topics ); + RdKafka::ErrorCode err = m_consumer->subscribe( topics ); if( err ) - CSP_THROW( RuntimeException, "Failed to subscribe to " << m_topics.size() << " topics: " << RdKafka::err2str( err ) ); + CSP_THROW( RuntimeException, + "Failed to subscribe to " << m_topics.size() << " topics: " << RdKafka::err2str( err ) ); - m_running = true; - m_pollThread = std::make_unique( [ this ](){ poll(); } ); + m_running = true; + m_pollThread = std::make_unique( [this]() { poll(); } ); } void KafkaConsumer::stop() @@ -157,18 +170,18 @@ void KafkaConsumer::stop() if( m_running ) { m_running = false; - m_pollThread -> join(); + m_pollThread->join(); } if( m_consumer.get() ) { - m_consumer -> close(); + m_consumer->close(); m_consumer.reset(); } } void KafkaConsumer::setNumPartitions( const std::string & topic, size_t num ) { - auto & topicData = m_topics[ topic ]; + auto & topicData = m_topics[topic]; topicData.partitionLive.resize( num, false ); } @@ -182,7 +195,7 @@ void KafkaConsumer::forceReplayCompleted() for( auto & subscriberEntry : topicData.subscribers ) { for( auto * subscriber : subscriberEntry.second ) - subscriber -> flagReplayComplete(); + subscriber->flagReplayComplete(); } topicData.flaggedReplayComplete = true; } @@ -195,71 +208,75 @@ void KafkaConsumer::poll() { while( m_running ) { - std::unique_ptr msg( m_consumer -> consume( m_mgr -> pollTimeoutMs() ) ); + std::unique_ptr msg( m_consumer->consume( m_mgr->pollTimeoutMs() ) ); - if( msg -> err() == RdKafka::ERR__TIMED_OUT ) + if( msg->err() == RdKafka::ERR__TIMED_OUT ) continue; - auto topicIt = m_topics.find( msg -> topic_name() ); + auto topicIt = m_topics.find( msg->topic_name() ); if( topicIt == m_topics.end() ) { - std::string errmsg = "KafkaConsumer: Message received on unknown topic: " + msg -> topic_name() + - " errcode: " + RdKafka::err2str( msg -> err() ) + " error: " + msg -> errstr(); - m_mgr -> pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); - - //We tend to accumulate more cases over time of error states that leave the engine deadlocked on PushPull adapters. - //This section is for cases where we get an error that is not topic specific, but is consumer specific, but we know its non-recoverable - //if it gets too long, or we realize that ANY error here should stop the engine, we can just always make it stop - if( msg -> err() == RdKafka::ERR_GROUP_AUTHORIZATION_FAILED ) - m_mgr -> forceShutdown( RdKafka::err2str( msg -> err() ) + " error: " + msg -> errstr() ); + std::string errmsg = "KafkaConsumer: Message received on unknown topic: " + msg->topic_name() + + " errcode: " + RdKafka::err2str( msg->err() ) + " error: " + msg->errstr(); + m_mgr->pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); + + // We tend to accumulate more cases over time of error states that leave the engine deadlocked on + // PushPull adapters. This section is for cases where we get an error that is not topic specific, but is + // consumer specific, but we know its non-recoverable if it gets too long, or we realize that ANY error + // here should stop the engine, we can just always make it stop + if( msg->err() == RdKafka::ERR_GROUP_AUTHORIZATION_FAILED ) + m_mgr->forceShutdown( RdKafka::err2str( msg->err() ) + " error: " + msg->errstr() ); continue; } - auto & topicData = topicIt -> second; + auto & topicData = topicIt->second; - if( msg -> err() == RdKafka::ERR_NO_ERROR && msg -> len() ) + if( msg->err() == RdKafka::ERR_NO_ERROR && msg->len() ) { - if( !msg -> key() ) + if( !msg->key() ) { - std::string errmsg = "KafkaConsumer: Message received with null key on topic " + msg -> topic_name() + "."; - m_mgr -> pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); + std::string errmsg + = "KafkaConsumer: Message received with null key on topic " + msg->topic_name() + "."; + m_mgr->pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); continue; } - //printf( "Msg %s:%s on %d\n", msg -> topic_name().c_str(), msg -> key() -> c_str(), msg -> partition() ); - auto subscribersIt = topicData.subscribers.find( *msg -> key() ); + // printf( "Msg %s:%s on %d\n", msg -> topic_name().c_str(), msg -> key() -> c_str(), msg -> partition() + // ); + auto subscribersIt = topicData.subscribers.find( *msg->key() ); if( subscribersIt == topicData.subscribers.end() && topicData.wildcardSubscriber == nullptr ) continue; auto & partitionLive = topicData.partitionLive; - if( ( uint32_t ) msg -> partition() >= partitionLive.size() ) - partitionLive.resize( msg -> partition() + 1, false ); + if( (uint32_t)msg->partition() >= partitionLive.size() ) + partitionLive.resize( msg->partition() + 1, false ); - bool live = partitionLive[ msg -> partition() ]; + bool live = partitionLive[msg->partition()]; if( subscribersIt != topicData.subscribers.end() ) { - for( auto it : subscribersIt -> second ) - it -> onMessage( msg.get(), live ); + for( auto it : subscribersIt->second ) + it->onMessage( msg.get(), live ); } - //Note we always have to tick wildcard as live because it can get messages from multiple - //partitions, some which may have done replaying and some not ( not to mention that data can be out of order ) + // Note we always have to tick wildcard as live because it can get messages from multiple + // partitions, some which may have done replaying and some not ( not to mention that data can be out of + // order ) if( topicData.wildcardSubscriber ) - topicData.wildcardSubscriber -> onMessage( msg.get(), true ); + topicData.wildcardSubscriber->onMessage( msg.get(), true ); } - //Not sure why, but it looks like we repeatedly get EOF callbacks even after the original one - //may want to look into this. Not an issue in practice, but seems like unnecessary overhead - else if( msg -> err() == RdKafka::ERR__PARTITION_EOF ) + // Not sure why, but it looks like we repeatedly get EOF callbacks even after the original one + // may want to look into this. Not an issue in practice, but seems like unnecessary overhead + else if( msg->err() == RdKafka::ERR__PARTITION_EOF ) { - auto const partition = msg -> partition(); + auto const partition = msg->partition(); auto & partitionLive = topicData.partitionLive; - if( ( uint32_t ) partition >= partitionLive.size() ) + if( (uint32_t)partition >= partitionLive.size() ) partitionLive.resize( partition + 1, false ); - partitionLive[ partition ] = true; + partitionLive[partition] = true; - //need this gaurd since we get this repeatedly after initial EOF + // need this gaurd since we get this repeatedly after initial EOF if( !topicData.flaggedReplayComplete ) { bool allDone = true; @@ -274,11 +291,12 @@ void KafkaConsumer::poll() if( allDone ) { - //we need to flag end in case the topic doesnt have any incoming data, we cant stall the engine on the pull side of the adapter + // we need to flag end in case the topic doesnt have any incoming data, we cant stall the engine + // on the pull side of the adapter for( auto & subscriberEntry : topicData.subscribers ) { for( auto * subscriber : subscriberEntry.second ) - subscriber -> flagReplayComplete(); + subscriber->flagReplayComplete(); } topicData.flaggedReplayComplete = true; } @@ -286,29 +304,30 @@ void KafkaConsumer::poll() } else { - //In most cases we should not get here, if we do then something is wrong - //safest bet is to release the pull adapter so it doesnt stall the engine and - //we can let the error msg through + // In most cases we should not get here, if we do then something is wrong + // safest bet is to release the pull adapter so it doesnt stall the engine and + // we can let the error msg through if( !topicData.flaggedReplayComplete ) { - //flag inputs as done so they dont hold up the engine + // flag inputs as done so they dont hold up the engine for( auto & subscriberEntry : topicData.subscribers ) { for( auto * subscriber : subscriberEntry.second ) - subscriber -> flagReplayComplete(); + subscriber->flagReplayComplete(); } topicData.flaggedReplayComplete = true; } - std::string errmsg = "KafkaConsumer: Message error on topic \"" + msg -> topic_name() + "\". errcode: " + RdKafka::err2str( msg -> err() ) + " error: " + msg -> errstr(); - m_mgr -> pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); + std::string errmsg = "KafkaConsumer: Message error on topic \"" + msg->topic_name() + + "\". errcode: " + RdKafka::err2str( msg->err() ) + " error: " + msg->errstr(); + m_mgr->pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); } } } catch( const Exception & err ) { - m_mgr -> rootEngine() -> shutdown( std::current_exception() ); + m_mgr->rootEngine()->shutdown( std::current_exception() ); } } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaInputAdapter.cpp b/cpp/csp/adapters/kafka/KafkaInputAdapter.cpp index 5d95176e..9e408959 100644 --- a/cpp/csp/adapters/kafka/KafkaInputAdapter.cpp +++ b/cpp/csp/adapters/kafka/KafkaInputAdapter.cpp @@ -5,119 +5,136 @@ namespace csp::adapters::kafka { -KafkaInputAdapter::KafkaInputAdapter( Engine *engine, CspTypePtr &type, - PushMode pushMode, PushGroup *group, - const Dictionary &properties) - : PushPullInputAdapter( engine, type, pushMode, group, - properties.get( "adjust_out_of_order_time") ) +KafkaInputAdapter::KafkaInputAdapter( Engine * engine, CspTypePtr & type, PushMode pushMode, PushGroup * group, + const Dictionary & properties ) + : PushPullInputAdapter( engine, type, pushMode, group, properties.get( "adjust_out_of_order_time" ) ) { - if( type -> type() != CspType::Type::STRUCT && - type -> type() != CspType::Type::STRING ) - CSP_THROW( RuntimeException, "Unsupported type: " << type -> type() ); - + if( type->type() != CspType::Type::STRUCT && type->type() != CspType::Type::STRING ) + CSP_THROW( RuntimeException, "Unsupported type: " << type->type() ); if( properties.exists( "meta_field_map" ) ) { const CspStructType & structType = static_cast( *type ); - const Dictionary & metaFieldMap = *properties.get( "meta_field_map" ); + const Dictionary & metaFieldMap = *properties.get( "meta_field_map" ); - if( !metaFieldMap.empty() && type -> type() != CspType::Type::STRUCT ) + if( !metaFieldMap.empty() && type->type() != CspType::Type::STRUCT ) CSP_THROW( ValueError, "meta_field_map is not supported on non-struct types" ); if( metaFieldMap.exists( "partition" ) ) { std::string partitionFieldName = metaFieldMap.get( "partition" ); - m_partitionField = structType.meta() -> field( partitionFieldName ); + m_partitionField = structType.meta()->field( partitionFieldName ); if( !m_partitionField ) - CSP_THROW( ValueError, "field " << partitionFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_partitionField -> type() -> type() != CspType::Type::INT64 ) - CSP_THROW( ValueError, "field " << partitionFieldName << " must be of type int on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << partitionFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_partitionField->type()->type() != CspType::Type::INT64 ) + CSP_THROW( ValueError, + "field " << partitionFieldName << " must be of type int on struct type " + << structType.meta()->name() ); } if( metaFieldMap.exists( "offset" ) ) { std::string offsetFieldName = metaFieldMap.get( "offset" ); - m_offsetField = structType.meta() -> field( offsetFieldName ); + m_offsetField = structType.meta()->field( offsetFieldName ); if( !m_offsetField ) - CSP_THROW( ValueError, "field " << offsetFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_offsetField -> type() -> type() != CspType::Type::INT64 ) - CSP_THROW( ValueError, "field " << offsetFieldName << " must be of type int on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << offsetFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_offsetField->type()->type() != CspType::Type::INT64 ) + CSP_THROW( ValueError, + "field " << offsetFieldName << " must be of type int on struct type " + << structType.meta()->name() ); } if( metaFieldMap.exists( "live" ) ) { std::string liveFieldName = metaFieldMap.get( "live" ); - m_liveField = structType.meta() -> field( liveFieldName ); + m_liveField = structType.meta()->field( liveFieldName ); if( !m_liveField ) - CSP_THROW( ValueError, "field " << liveFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_liveField -> type() -> type() != CspType::Type::BOOL ) - CSP_THROW( ValueError, "field " << liveFieldName << " must be of type bool on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << liveFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_liveField->type()->type() != CspType::Type::BOOL ) + CSP_THROW( ValueError, + "field " << liveFieldName << " must be of type bool on struct type " + << structType.meta()->name() ); } if( metaFieldMap.exists( "timestamp" ) ) { std::string timestampFieldName = metaFieldMap.get( "timestamp" ); - m_timestampField = structType.meta() -> field( timestampFieldName ); + m_timestampField = structType.meta()->field( timestampFieldName ); if( !m_timestampField ) - CSP_THROW( ValueError, "field " << timestampFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_timestampField -> type() -> type() != CspType::Type::DATETIME ) - CSP_THROW( ValueError, "field " << timestampFieldName << " must be of type datetime on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << timestampFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_timestampField->type()->type() != CspType::Type::DATETIME ) + CSP_THROW( ValueError, + "field " << timestampFieldName << " must be of type datetime on struct type " + << structType.meta()->name() ); } if( metaFieldMap.exists( "key" ) ) { std::string keyFieldName = metaFieldMap.get( "key" ); - m_keyField = structType.meta() -> field( keyFieldName ); + m_keyField = structType.meta()->field( keyFieldName ); if( !m_keyField ) - CSP_THROW( ValueError, "field " << keyFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_keyField -> type() -> type() != CspType::Type::STRING ) - CSP_THROW( ValueError, "field " << keyFieldName << " must be of type string on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << keyFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_keyField->type()->type() != CspType::Type::STRING ) + CSP_THROW( ValueError, + "field " << keyFieldName << " must be of type string on struct type " + << structType.meta()->name() ); } } m_converter = utils::MessageStructConverterCache::instance().create( type, properties ); } -void KafkaInputAdapter::processMessage( RdKafka::Message* message, bool live, csp::PushBatch* batch ) +void KafkaInputAdapter::processMessage( RdKafka::Message * message, bool live, csp::PushBatch * batch ) { bool pushLive = live || flaggedLive(); DateTime msgTime; - auto ts = message -> timestamp(); + auto ts = message->timestamp(); if( ts.type != RdKafka::MessageTimestamp::MSG_TIMESTAMP_NOT_AVAILABLE ) { msgTime = DateTime::fromMilliseconds( ts.timestamp ); - //If user requested kafka data earlier than engine start, we will force it as live so it makes it into the engine - if( msgTime < rootEngine() -> startTime() ) + // If user requested kafka data earlier than engine start, we will force it as live so it makes it into the + // engine + if( msgTime < rootEngine()->startTime() ) pushLive = true; } else { - //if we cant extract time from the msg we cant push sim, so make force it live/realtime + // if we cant extract time from the msg we cant push sim, so make force it live/realtime pushLive = true; } - if( type() -> type() == CspType::Type::STRUCT ) + if( type()->type() == CspType::Type::STRUCT ) { - auto tick = m_converter -> asStruct( message -> payload(), message -> len() ); + auto tick = m_converter->asStruct( message->payload(), message->len() ); if( m_partitionField ) - m_partitionField -> setValue( tick.get(), message -> partition() ); + m_partitionField->setValue( tick.get(), message->partition() ); if( m_offsetField ) - m_offsetField -> setValue( tick.get(), message -> offset() ); + m_offsetField->setValue( tick.get(), message->offset() ); if( m_liveField ) - m_liveField -> setValue( tick.get(), live ); + m_liveField->setValue( tick.get(), live ); if( m_timestampField && !msgTime.isNone() ) - m_timestampField -> setValue( tick.get(), msgTime ); + m_timestampField->setValue( tick.get(), msgTime ); if( m_keyField ) - m_keyField -> setValue( tick.get(), *message -> key() ); + m_keyField->setValue( tick.get(), *message->key() ); pushTick( pushLive, msgTime, std::move( tick ), batch ); } - else if( type() -> type() == CspType::Type::STRING ) + else if( type()->type() == CspType::Type::STRING ) { - pushTick( pushLive, msgTime, std::string( ( const char * ) message -> payload(), message -> len() ) ); + pushTick( pushLive, msgTime, std::string( (const char *)message->payload(), message->len() ) ); } } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaOutputAdapter.cpp b/cpp/csp/adapters/kafka/KafkaOutputAdapter.cpp index 4f453d46..f4a87060 100644 --- a/cpp/csp/adapters/kafka/KafkaOutputAdapter.cpp +++ b/cpp/csp/adapters/kafka/KafkaOutputAdapter.cpp @@ -4,54 +4,55 @@ namespace csp::adapters::kafka { -KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, - CspTypePtr & type, const Dictionary & properties ) : OutputAdapter( engine ), - m_publisher( publisher ) +KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, + const Dictionary & properties ) + : OutputAdapter( engine ) + , m_publisher( publisher ) { if( !publisher.isRawBytes() ) - m_dataMapper = utils::OutputDataMapperCache::instance().create( type, *properties.get( "field_map" ) ); - else if( type -> type() != CspType::Type::STRING ) - CSP_THROW( TypeError, "RAW_BYTES output expected ts[str] got ts type " << type -> type() ); + m_dataMapper + = utils::OutputDataMapperCache::instance().create( type, *properties.get( "field_map" ) ); + else if( type->type() != CspType::Type::STRING ) + CSP_THROW( TypeError, "RAW_BYTES output expected ts[str] got ts type " << type->type() ); } -KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, - CspTypePtr & type, const Dictionary & properties, - const std::string & key ) : KafkaOutputAdapter( engine, publisher, type, properties ) +KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, + const Dictionary & properties, const std::string & key ) + : KafkaOutputAdapter( engine, publisher, type, properties ) { m_publisher.setKey( key ); } -KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, - CspTypePtr &type, const Dictionary & properties, - const std::vector & keyFields ) : KafkaOutputAdapter( engine, publisher, type, properties ) +KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, + const Dictionary & properties, const std::vector & keyFields ) + : KafkaOutputAdapter( engine, publisher, type, properties ) { addFields( keyFields, type ); } -KafkaOutputAdapter::~KafkaOutputAdapter() -{ -} +KafkaOutputAdapter::~KafkaOutputAdapter() {} void KafkaOutputAdapter::addFields( const std::vector & keyFields, CspTypePtr & type, size_t i ) { - std::string fieldName = keyFields[ i ]; - auto structMeta = static_cast( *type ).meta(); - auto structField = structMeta -> field( fieldName ); + std::string fieldName = keyFields[i]; + auto structMeta = static_cast( *type ).meta(); + auto structField = structMeta->field( fieldName ); if( !structField ) - CSP_THROW( InvalidArgument, "Struct type " << structMeta -> name() << " missing required field " << fieldName ); + CSP_THROW( InvalidArgument, "Struct type " << structMeta->name() << " missing required field " << fieldName ); if( i == keyFields.size() - 1 ) { - CSP_TRUE_OR_THROW_RUNTIME( structField -> type() -> type() == CspType::Type::STRING, - "Key field must be of type string, got " << structField -> type() -> type().asString() ); + CSP_TRUE_OR_THROW_RUNTIME( structField->type()->type() == CspType::Type::STRING, + "Key field must be of type string, got " << structField->type()->type().asString() ); m_structFields.emplace_back( structField ); } else { - CSP_TRUE_OR_THROW_RUNTIME( structField -> type() -> type() == CspType::Type::STRUCT, - "Non-key field must be of type struct, got " << structField -> type() -> type().asString() ); + CSP_TRUE_OR_THROW_RUNTIME( structField->type()->type() == CspType::Type::STRUCT, + "Non-key field must be of type struct, got " + << structField->type()->type().asString() ); m_structFields.emplace_back( structField ); - auto t = structField -> type(); + auto t = structField->type(); addFields( keyFields, t, i + 1 ); } } @@ -59,28 +60,28 @@ void KafkaOutputAdapter::addFields( const std::vector & keyFields, const std::string & KafkaOutputAdapter::getKey( const Struct * struct_ ) { for( size_t i = 0; i < m_structFields.size() - 1; ++i ) - struct_ = m_structFields[ i ] -> value( struct_ ).get(); + struct_ = m_structFields[i]->value( struct_ ).get(); - return m_structFields[ m_structFields.size() - 1 ] -> value( struct_ ); + return m_structFields[m_structFields.size() - 1]->value( struct_ ); } void KafkaOutputAdapter::executeImpl() { if( m_publisher.isRawBytes() ) { - const std::string & value = input() -> lastValueTyped(); + const std::string & value = input()->lastValueTyped(); m_publisher.send( value.c_str(), value.length() ); } else { auto & msgWriter = m_publisher.msgWriter(); msgWriter.processTick( *m_dataMapper, input() ); - + if( !m_structFields.empty() ) - m_publisher.setKey( getKey( input() -> lastValueTyped().get() ) ); - + m_publisher.setKey( getKey( input()->lastValueTyped().get() ) ); + m_publisher.scheduleEndCycleEvent(); } } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaPublisher.cpp b/cpp/csp/adapters/kafka/KafkaPublisher.cpp index 8678b01f..cf58f687 100644 --- a/cpp/csp/adapters/kafka/KafkaPublisher.cpp +++ b/cpp/csp/adapters/kafka/KafkaPublisher.cpp @@ -1,17 +1,18 @@ -#include #include #include -#include #include +#include +#include #include namespace csp::adapters::kafka { -KafkaPublisher::KafkaPublisher( KafkaAdapterManager * mgr, const Dictionary & properties, std::string topic ) : m_adapterMgr( *mgr ), - m_engine( mgr -> engine() ), - m_topic( std::move( topic ) ) +KafkaPublisher::KafkaPublisher( KafkaAdapterManager * mgr, const Dictionary & properties, std::string topic ) + : m_adapterMgr( *mgr ) + , m_engine( mgr->engine() ) + , m_topic( std::move( topic ) ) { utils::MsgProtocol protocol = utils::MsgProtocol( properties.get( "protocol" ) ); @@ -25,40 +26,40 @@ KafkaPublisher::KafkaPublisher( KafkaAdapterManager * mgr, const Dictionary & pr break; default: - CSP_THROW( NotImplemented, "msg protocol " << protocol << " not currently supported for kafka output adapters" ); + CSP_THROW( NotImplemented, + "msg protocol " << protocol << " not currently supported for kafka output adapters" ); break; } } -KafkaPublisher::~KafkaPublisher() -{ -} +KafkaPublisher::~KafkaPublisher() {} -OutputAdapter * KafkaPublisher::getOutputAdapter( CspTypePtr & type, const Dictionary & properties, const std::string & key ) +OutputAdapter * KafkaPublisher::getOutputAdapter( CspTypePtr & type, const Dictionary & properties, + const std::string & key ) { if( isRawBytes() && !m_adapters.empty() ) - CSP_THROW( RuntimeException, "Attempting to publish multiple timeseries to kafka key " << key << " with RAW_BYTES protocol. Only one output per key is allowed" ); + CSP_THROW( RuntimeException, + "Attempting to publish multiple timeseries to kafka key " + << key << " with RAW_BYTES protocol. Only one output per key is allowed" ); - //create and register adapter - auto adapter = m_engine -> createOwnedObject( *this, type, properties, key ); + // create and register adapter + auto adapter = m_engine->createOwnedObject( *this, type, properties, key ); m_adapters.emplace_back( adapter ); - return m_adapters[ m_adapters.size() - 1 ]; + return m_adapters[m_adapters.size() - 1]; } -OutputAdapter * KafkaPublisher::getOutputAdapter( CspTypePtr & type, const Dictionary & properties, const std::vector & keyFields ) +OutputAdapter * KafkaPublisher::getOutputAdapter( CspTypePtr & type, const Dictionary & properties, + const std::vector & keyFields ) { if( isRawBytes() ) CSP_THROW( RuntimeException, "vector of key fields is unsupported for RAW_BYTES protocol" ); - auto adapter = m_engine -> createOwnedObject( *this, type, properties, keyFields ); + auto adapter = m_engine->createOwnedObject( *this, type, properties, keyFields ); m_adapters.emplace_back( adapter ); - return m_adapters[ m_adapters.size() - 1 ]; + return m_adapters[m_adapters.size() - 1]; } -PushInputAdapter * KafkaPublisher::getStatusAdapter() -{ - return nullptr; -} +PushInputAdapter * KafkaPublisher::getStatusAdapter() { return nullptr; } void KafkaPublisher::start( std::shared_ptr producer ) { @@ -67,24 +68,26 @@ void KafkaPublisher::start( std::shared_ptr producer ) std::unique_ptr tconf( RdKafka::Conf::create( RdKafka::Conf::CONF_TOPIC ) ); std::string errstr; - m_kafkaTopic = std::shared_ptr( RdKafka::Topic::create( m_producer.get(), m_topic, tconf.get(), errstr ) ); + m_kafkaTopic + = std::shared_ptr( RdKafka::Topic::create( m_producer.get(), m_topic, tconf.get(), errstr ) ); if( !m_kafkaTopic ) - CSP_THROW( RuntimeException, "Failed to create RdKafka::Topic for producer on topic " << m_topic << ":" << errstr ); + CSP_THROW( RuntimeException, + "Failed to create RdKafka::Topic for producer on topic " << m_topic << ":" << errstr ); } void KafkaPublisher::stop() { - //Note its important to release the topic here because its underlying mem lives in the kafka context of the Producer - //if Producer is destroyed first, when we try to destroy the topic its a memory error + // Note its important to release the topic here because its underlying mem lives in the kafka context of the + // Producer if Producer is destroyed first, when we try to destroy the topic its a memory error m_kafkaTopic.reset(); } void KafkaPublisher::send( const void * data, size_t len ) { RdKafka::ErrorCode err; - err = m_producer -> produce( m_kafkaTopic.get(), RdKafka::Topic::PARTITION_UA, RdKafka::Producer::RK_MSG_COPY, - const_cast( data ), len, m_key.c_str(), m_key.length(), nullptr ); - + err = m_producer->produce( m_kafkaTopic.get(), RdKafka::Topic::PARTITION_UA, RdKafka::Producer::RK_MSG_COPY, + const_cast( data ), len, m_key.c_str(), m_key.length(), nullptr ); + if( err != RdKafka::ERR_NO_ERROR ) { std::string errMsg( "KafkaPublisher Error sending message " ); @@ -95,8 +98,8 @@ void KafkaPublisher::send( const void * data, size_t len ) void KafkaPublisher::onEndCycle() { - auto [data,len] = m_msgWriter -> finalize(); + auto [data, len] = m_msgWriter->finalize(); send( data, len ); } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaSubscriber.cpp b/cpp/csp/adapters/kafka/KafkaSubscriber.cpp index 75734ea3..092b8448 100644 --- a/cpp/csp/adapters/kafka/KafkaSubscriber.cpp +++ b/cpp/csp/adapters/kafka/KafkaSubscriber.cpp @@ -1,38 +1,38 @@ -#include #include +#include #include #include namespace csp::adapters::kafka { -KafkaSubscriber::KafkaSubscriber( KafkaAdapterManager * mgr, const Dictionary & properties ) : m_adapterMgr( *mgr ), - m_engine( mgr -> engine() ) +KafkaSubscriber::KafkaSubscriber( KafkaAdapterManager * mgr, const Dictionary & properties ) + : m_adapterMgr( *mgr ) + , m_engine( mgr->engine() ) { } -KafkaSubscriber::~KafkaSubscriber() -{ -} +KafkaSubscriber::~KafkaSubscriber() {} -PushInputAdapter * KafkaSubscriber::getInputAdapter( CspTypePtr & type, PushMode pushMode, const Dictionary & properties ) +PushInputAdapter * KafkaSubscriber::getInputAdapter( CspTypePtr & type, PushMode pushMode, + const Dictionary & properties ) { - //create and register adapter - auto adapter = m_engine -> createOwnedObject( type, pushMode, &m_pushGroup, properties ); + // create and register adapter + auto adapter = m_engine->createOwnedObject( type, pushMode, &m_pushGroup, properties ); m_adapters.emplace_back( adapter ); return adapter; } -void KafkaSubscriber::onMessage( RdKafka::Message* message, bool live ) +void KafkaSubscriber::onMessage( RdKafka::Message * message, bool live ) { - csp::PushBatch batch( m_engine -> rootEngine() ); + csp::PushBatch batch( m_engine->rootEngine() ); for( auto & adapter : m_adapters ) { try { - adapter -> processMessage( message, live, &batch ); + adapter->processMessage( message, live, &batch ); } catch( csp::Exception & err ) { @@ -44,7 +44,7 @@ void KafkaSubscriber::onMessage( RdKafka::Message* message, bool live ) void KafkaSubscriber::flagReplayComplete() { for( auto & adapter : m_adapters ) - adapter -> flagReplayComplete(); + adapter->flagReplayComplete(); } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.cpp b/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.cpp index 382f13ac..57f3c19b 100644 --- a/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.cpp +++ b/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.cpp @@ -1,26 +1,22 @@ -#include -#include #include #include +#include +#include #include namespace csp::adapters::parquet { -ArrowIPCFileReaderWrapper::~ArrowIPCFileReaderWrapper() -{ - close(); -} +ArrowIPCFileReaderWrapper::~ArrowIPCFileReaderWrapper() { close(); } -void ArrowIPCFileReaderWrapper::open( const std::string &fileName ) +void ArrowIPCFileReaderWrapper::open( const std::string & fileName ) { FileReaderWrapper::open( fileName ); try { - PARQUET_ASSIGN_OR_THROW( m_fileReader, - ::arrow::ipc::RecordBatchStreamReader::Open( m_inputFile ) ); + PARQUET_ASSIGN_OR_THROW( m_fileReader, ::arrow::ipc::RecordBatchStreamReader::Open( m_inputFile ) ); } - catch( std::exception &e ) + catch( std::exception & e ) { FileReaderWrapper::close(); CSP_THROW( RuntimeException, "Failed to open " << fileName << ":" << e.what() ); @@ -33,10 +29,11 @@ void ArrowIPCFileReaderWrapper::close() FileReaderWrapper::close(); } -bool ArrowIPCFileReaderWrapper::readNextRowGroup( const std::vector neededColumns, std::shared_ptr<::arrow::Table> &dst ) +bool ArrowIPCFileReaderWrapper::readNextRowGroup( const std::vector neededColumns, + std::shared_ptr<::arrow::Table> & dst ) { std::shared_ptr recordBatch; - STATUS_OK_OR_THROW_RUNTIME( m_fileReader -> ReadNext( &recordBatch ), "Failed to read next record batch" ); + STATUS_OK_OR_THROW_RUNTIME( m_fileReader->ReadNext( &recordBatch ), "Failed to read next record batch" ); if( recordBatch == nullptr ) { dst = nullptr; @@ -44,22 +41,18 @@ bool ArrowIPCFileReaderWrapper::readNextRowGroup( const std::vector neededC } std::vector> tableColumns; std::vector> tableColumnSpecs; - tableColumns.reserve(neededColumns.size()); - tableColumnSpecs.reserve(neededColumns.size()); + tableColumns.reserve( neededColumns.size() ); + tableColumnSpecs.reserve( neededColumns.size() ); - for( auto &&columnIndex : neededColumns ) + for( auto && columnIndex : neededColumns ) { - tableColumns.push_back(recordBatch->column(columnIndex)); - tableColumnSpecs.push_back(recordBatch->schema()->field(columnIndex)); + tableColumns.push_back( recordBatch->column( columnIndex ) ); + tableColumnSpecs.push_back( recordBatch->schema()->field( columnIndex ) ); } dst = arrow::Table::Make( ::arrow::schema( tableColumnSpecs ), tableColumns ); return true; } +void ArrowIPCFileReaderWrapper::getSchema( std::shared_ptr<::arrow::Schema> & dst ) { dst = m_fileReader->schema(); } -void ArrowIPCFileReaderWrapper::getSchema( std::shared_ptr<::arrow::Schema> &dst ) -{ - dst = m_fileReader -> schema(); -} - -} \ No newline at end of file +} // namespace csp::adapters::parquet \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp b/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp index daf23337..f4e36f3c 100644 --- a/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp +++ b/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp @@ -1,26 +1,24 @@ -#include -#include #include -#include #include +#include +#include +#include #include namespace csp::adapters::parquet { -void ArrowIPCFileWriterWrapper::openImpl( const std::string &fileName, const std::string &compression ) +void ArrowIPCFileWriterWrapper::openImpl( const std::string & fileName, const std::string & compression ) { CSP_TRUE_OR_THROW_RUNTIME( m_outputStream == nullptr, "Trying to open parquet file while previous was not closed" ); - PARQUET_ASSIGN_OR_THROW( - m_outputStream, - arrow::io::FileOutputStream::Open( fileName.c_str())); + PARQUET_ASSIGN_OR_THROW( m_outputStream, arrow::io::FileOutputStream::Open( fileName.c_str() ) ); arrow::ipc::IpcWriteOptions writeOptions; - writeOptions.codec = resolveCompressionCodec( compression ); + writeOptions.codec = resolveCompressionCodec( compression ); STATUS_OK_OR_THROW_RUNTIME( - ::arrow::ipc::MakeStreamWriter( m_outputStream.get(), getSchema(), writeOptions ).Value(&m_fileWriter), - "Failed to open arrow file writer" ); + ::arrow::ipc::MakeStreamWriter( m_outputStream.get(), getSchema(), writeOptions ).Value( &m_fileWriter ), + "Failed to open arrow file writer" ); } void ArrowIPCFileWriterWrapper::close() @@ -29,7 +27,7 @@ void ArrowIPCFileWriterWrapper::close() { // Let's move them first, if there are any exceptions, we still want the pointer to be null std::shared_ptr<::arrow::io::FileOutputStream> outputStream{ std::move( m_outputStream ) }; - std::shared_ptr<::arrow::ipc::RecordBatchWriter> fileWriter{ std::move( m_fileWriter ) }; + std::shared_ptr<::arrow::ipc::RecordBatchWriter> fileWriter{ std::move( m_fileWriter ) }; // Should be done by move constructor but let's be safe: m_outputStream = nullptr; m_fileWriter = nullptr; @@ -41,9 +39,10 @@ void ArrowIPCFileWriterWrapper::close() } } -void ArrowIPCFileWriterWrapper::writeTable( const std::shared_ptr<::arrow::Table> &table ) +void ArrowIPCFileWriterWrapper::writeTable( const std::shared_ptr<::arrow::Table> & table ) { - STATUS_OK_OR_THROW_RUNTIME( m_fileWriter->WriteTable( *table, table->num_rows()), "Failed to write to arrow file" ); + STATUS_OK_OR_THROW_RUNTIME( m_fileWriter->WriteTable( *table, table->num_rows() ), + "Failed to write to arrow file" ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/FileReaderWrapper.cpp b/cpp/csp/adapters/parquet/FileReaderWrapper.cpp index 682883b9..0b0b21c2 100644 --- a/cpp/csp/adapters/parquet/FileReaderWrapper.cpp +++ b/cpp/csp/adapters/parquet/FileReaderWrapper.cpp @@ -1,25 +1,19 @@ -#include #include +#include #include - namespace csp::adapters::parquet { -FileReaderWrapper::~FileReaderWrapper() -{ - close(); -} +FileReaderWrapper::~FileReaderWrapper() { close(); } -void FileReaderWrapper::open( const std::string &fileName ) +void FileReaderWrapper::open( const std::string & fileName ) { if( m_inputFile ) { close(); } - PARQUET_ASSIGN_OR_THROW( - m_inputFile, - arrow::io::ReadableFile::Open( fileName ) ); + PARQUET_ASSIGN_OR_THROW( m_inputFile, arrow::io::ReadableFile::Open( fileName ) ); m_fileName = fileName; } @@ -32,4 +26,4 @@ void FileReaderWrapper::close() } } -} \ No newline at end of file +} // namespace csp::adapters::parquet \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/FileWriterWrapper.cpp b/cpp/csp/adapters/parquet/FileWriterWrapper.cpp index 02e84631..71e22d28 100644 --- a/cpp/csp/adapters/parquet/FileWriterWrapper.cpp +++ b/cpp/csp/adapters/parquet/FileWriterWrapper.cpp @@ -2,48 +2,48 @@ #include #include #include -#include #include +#include namespace csp::adapters::parquet { std::unordered_map FileWriterWrapper::m_compressionNameMapping{ - { "", ::arrow::Compression::UNCOMPRESSED }, - { "snappy", ::arrow::Compression::SNAPPY }, - { "gzip", ::arrow::Compression::GZIP }, - { "brotli", ::arrow::Compression::BROTLI }, - { "zstd", ::arrow::Compression::ZSTD }, - { "lz4", ::arrow::Compression::LZ4 }, - { "lz4_frame", ::arrow::Compression::LZ4_FRAME }, - { "lzo", ::arrow::Compression::LZO }, - { "bz2", ::arrow::Compression::BZ2 } -}; + { "", ::arrow::Compression::UNCOMPRESSED }, + { "snappy", ::arrow::Compression::SNAPPY }, + { "gzip", ::arrow::Compression::GZIP }, + { "brotli", ::arrow::Compression::BROTLI }, + { "zstd", ::arrow::Compression::ZSTD }, + { "lz4", ::arrow::Compression::LZ4 }, + { "lz4_frame", ::arrow::Compression::LZ4_FRAME }, + { "lzo", ::arrow::Compression::LZO }, + { "bz2", ::arrow::Compression::BZ2 } }; -void FileWriterWrapper::open( const std::string &fileName, const std::string &compression, bool allowOverwrite ) +void FileWriterWrapper::open( const std::string & fileName, const std::string & compression, bool allowOverwrite ) { if( !allowOverwrite ) { CSP_TRUE_OR_THROW_RUNTIME( !utils::fileExists( fileName ), - "Trying to overwrite existing file " << fileName << " while allow_overwrite is false" ); + "Trying to overwrite existing file " << fileName + << " while allow_overwrite is false" ); } utils::mkdir( utils::dirname( fileName ) ); openImpl( fileName, compression ); } -::arrow::Compression::type FileWriterWrapper::resolveCompression( const std::string &compression ) +::arrow::Compression::type FileWriterWrapper::resolveCompression( const std::string & compression ) { auto it = m_compressionNameMapping.find( compression ); CSP_TRUE_OR_THROW_RUNTIME( it != m_compressionNameMapping.end(), "Unable to resolve compression: " << compression ); return it->second; } -std::unique_ptr FileWriterWrapper::resolveCompressionCodec( const std::string &compression ) +std::unique_ptr FileWriterWrapper::resolveCompressionCodec( const std::string & compression ) { auto compressionType = resolveCompression( compression ); - auto res = arrow::util::Codec::Create( compressionType ); - STATUS_OK_OR_THROW_RUNTIME(res.status(), "Failed to create arrow codec for " << compressionType ); - return std::move(res.ValueUnsafe()); + auto res = arrow::util::Codec::Create( compressionType ); + STATUS_OK_OR_THROW_RUNTIME( res.status(), "Failed to create arrow codec for " << compressionType ); + return std::move( res.ValueUnsafe() ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp b/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp index 3b103283..6bd81e52 100644 --- a/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp +++ b/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp @@ -1,20 +1,21 @@ -#include -#include +#include +#include +#include #include +#include #include #include #include -#include -#include -#include -#include +#include #include +#include namespace csp::adapters::parquet { -FileWriterWrapperContainer::WriterPtr FileWriterWrapperContainer::createSingleFileWrapper( const std::shared_ptr &schema, - bool isWriteArrowBinary ) +FileWriterWrapperContainer::WriterPtr +FileWriterWrapperContainer::createSingleFileWrapper( const std::shared_ptr & schema, + bool isWriteArrowBinary ) { if( isWriteArrowBinary ) { @@ -26,76 +27,78 @@ FileWriterWrapperContainer::WriterPtr FileWriterWrapperContainer::createSingleFi } } -SingleFileWriterWrapperContainer::SingleFileWriterWrapperContainer( std::shared_ptr schema, bool isWriteArrowBinary ) - : m_fileWriterWrapper( createSingleFileWrapper( schema, isWriteArrowBinary ) ) +SingleFileWriterWrapperContainer::SingleFileWriterWrapperContainer( std::shared_ptr schema, + bool isWriteArrowBinary ) + : m_fileWriterWrapper( createSingleFileWrapper( schema, isWriteArrowBinary ) ) { } -void SingleFileWriterWrapperContainer::SingleFileWriterWrapperContainer::open( const std::string &fileName, const std::string &compression, +void SingleFileWriterWrapperContainer::SingleFileWriterWrapperContainer::open( const std::string & fileName, + const std::string & compression, bool allowOverwrite ) { - m_fileWriterWrapper -> open( fileName, compression, allowOverwrite ); - setOpen(true); + m_fileWriterWrapper->open( fileName, compression, allowOverwrite ); + setOpen( true ); } void SingleFileWriterWrapperContainer::close() { - m_fileWriterWrapper -> close(); - setOpen(false); + m_fileWriterWrapper->close(); + setOpen( false ); } -void SingleFileWriterWrapperContainer::writeData( const std::vector> &columnBuilders ) +void SingleFileWriterWrapperContainer::writeData( + const std::vector> & columnBuilders ) { std::vector> columns; columns.reserve( columnBuilders.size() ); - for( auto &&columnBuilder:columnBuilders ) + for( auto && columnBuilder : columnBuilders ) { - columns.push_back( columnBuilder -> buildArray() ); + columns.push_back( columnBuilder->buildArray() ); } - auto table = arrow::Table::Make( m_fileWriterWrapper -> getSchema(), columns ); + auto table = arrow::Table::Make( m_fileWriterWrapper->getSchema(), columns ); - m_fileWriterWrapper -> writeTable( table ); + m_fileWriterWrapper->writeTable( table ); } - -MultipleFileWriterWrapperContainer::MultipleFileWriterWrapperContainer( std::shared_ptr schema, bool isWriteArrowBinary ) +MultipleFileWriterWrapperContainer::MultipleFileWriterWrapperContainer( std::shared_ptr schema, + bool isWriteArrowBinary ) { - auto &fields = schema -> fields(); + auto & fields = schema->fields(); m_fileWriterWrappers.reserve( fields.size() ); - for( auto &&field:fields ) + for( auto && field : fields ) { std::vector> curFields; - std::string fileExtension = isWriteArrowBinary ? ".arrow" : ".parquet"; - std::string fileName = field -> name() + fileExtension; + std::string fileExtension = isWriteArrowBinary ? ".arrow" : ".parquet"; + std::string fileName = field->name() + fileExtension; - m_fileWriterWrappers.push_back( { fileName, - createSingleFileWrapper( arrow::schema( { field } ), isWriteArrowBinary ) } ); + m_fileWriterWrappers.push_back( + { fileName, createSingleFileWrapper( arrow::schema( { field } ), isWriteArrowBinary ) } ); } } -void MultipleFileWriterWrapperContainer::open( const std::string &fileName, const std::string &compression, bool allowOverwrite ) +void MultipleFileWriterWrapperContainer::open( const std::string & fileName, const std::string & compression, + bool allowOverwrite ) { - for( auto &&record : m_fileWriterWrappers ) + for( auto && record : m_fileWriterWrappers ) { - record.m_fileWriterWrapper -> open( fileName + '/' + record.m_fileName, - compression, - allowOverwrite ); - + record.m_fileWriterWrapper->open( fileName + '/' + record.m_fileName, compression, allowOverwrite ); } - setOpen(true); + setOpen( true ); } void MultipleFileWriterWrapperContainer::close() { - for( auto &&record : m_fileWriterWrappers ) + for( auto && record : m_fileWriterWrappers ) { - record.m_fileWriterWrapper -> close(); + record.m_fileWriterWrapper->close(); } - setOpen(false); + setOpen( false ); } -void MultipleFileWriterWrapperContainer::writeData( const std::vector> &columnBuilders ) +void MultipleFileWriterWrapperContainer::writeData( + const std::vector> & columnBuilders ) { std::vector> columns; columns.reserve( 1 ); @@ -105,11 +108,11 @@ void MultipleFileWriterWrapperContainer::writeData( const std::vector buildArray() ); - auto &fileWriterWrapper = m_fileWriterWrappers[ i ].m_fileWriterWrapper; - auto table = arrow::Table::Make( fileWriterWrapper -> getSchema(), columns ); - fileWriterWrapper -> writeTable( table ); + columns.push_back( columnBuilders[i]->buildArray() ); + auto & fileWriterWrapper = m_fileWriterWrappers[i].m_fileWriterWrapper; + auto table = arrow::Table::Make( fileWriterWrapper->getSchema(), columns ); + fileWriterWrapper->writeTable( table ); } } -} \ No newline at end of file +} // namespace csp::adapters::parquet \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp index 325dd249..19e9d5c6 100644 --- a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp +++ b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp @@ -4,11 +4,10 @@ namespace csp::adapters::parquet { -ParquetDictBasketOutputWriter::ParquetDictBasketOutputWriter( - ParquetOutputAdapterManager *outputAdapterManager, - const std::string &columnName ) - : - ParquetWriter( outputAdapterManager, false ), m_nextCycleIndex( 0 ) +ParquetDictBasketOutputWriter::ParquetDictBasketOutputWriter( ParquetOutputAdapterManager * outputAdapterManager, + const std::string & columnName ) + : ParquetWriter( outputAdapterManager, false ) + , m_nextCycleIndex( 0 ) { m_symbolOutputAdapter = getScalarOutputHandler( CspType::STRING(), columnName + "__csp_symbol" ); m_cycleIndexOutputAdapter = createScalarOutputHandler( CspType::UINT16(), columnName + "__csp_value_count" ); @@ -18,127 +17,124 @@ void ParquetDictBasketOutputWriter::start() { ParquetWriter::start(); m_indexFileWriterContainer = std::make_unique( - arrow::schema( { arrow::field( m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getColumnName(), - m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getDataType() ) } ), - m_adapterMgr.isWriteArrowBinary() ); + arrow::schema( { arrow::field( m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 )->getColumnName(), + m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 )->getDataType() ) } ), + m_adapterMgr.isWriteArrowBinary() ); if( !m_adapterMgr.getFileName().empty() ) { - m_indexFileWriterContainer -> open( m_adapterMgr.getFileName(), - m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); - + m_indexFileWriterContainer->open( m_adapterMgr.getFileName(), m_adapterMgr.getCompression(), + m_adapterMgr.isAllowOverwrite() ); } } void ParquetDictBasketOutputWriter::stop() { - auto &&indexColumnArrayBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ); - if( indexColumnArrayBuilder -> length() > 0 ) + auto && indexColumnArrayBuilder = m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 ); + if( indexColumnArrayBuilder->length() > 0 ) { - CSP_TRUE_OR_THROW_RUNTIME(isFileOpen(), "On stop ParquetDictBasketOutputWriter has data to write but no open file"); - m_indexFileWriterContainer -> writeData( { indexColumnArrayBuilder } ); + CSP_TRUE_OR_THROW_RUNTIME( isFileOpen(), + "On stop ParquetDictBasketOutputWriter has data to write but no open file" ); + m_indexFileWriterContainer->writeData( { indexColumnArrayBuilder } ); } - m_indexFileWriterContainer -> close(); + m_indexFileWriterContainer->close(); m_indexFileWriterContainer = nullptr; ParquetWriter::stop(); } -void ParquetDictBasketOutputWriter::writeValue( const std::string &valueKey, const TimeSeriesProvider *ts ) +void ParquetDictBasketOutputWriter::writeValue( const std::string & valueKey, const TimeSeriesProvider * ts ) { m_adapterMgr.scheduleEndCycle(); - m_symbolOutputAdapter -> writeValue( valueKey ); + m_symbolOutputAdapter->writeValue( valueKey ); ParquetWriter::onEndCycle(); ++m_nextCycleIndex; } - void ParquetDictBasketOutputWriter::onEndCycle() { - if(isFileOpen()) + if( isFileOpen() ) { - m_cycleIndexOutputAdapter -> writeValue( m_nextCycleIndex ); - auto &&indexColumnArrayBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ); - indexColumnArrayBuilder -> handleRowFinished(); - if( indexColumnArrayBuilder -> length() >= getChunkSize() ) + m_cycleIndexOutputAdapter->writeValue( m_nextCycleIndex ); + auto && indexColumnArrayBuilder = m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 ); + indexColumnArrayBuilder->handleRowFinished(); + if( indexColumnArrayBuilder->length() >= getChunkSize() ) { - m_indexFileWriterContainer -> writeData( { indexColumnArrayBuilder } ); + m_indexFileWriterContainer->writeData( { indexColumnArrayBuilder } ); } m_nextCycleIndex = 0; } else { - CSP_TRUE_OR_THROW_RUNTIME(m_nextCycleIndex==0, "ParquetDictBasketOutputWriter has non 0 index with no open file"); + CSP_TRUE_OR_THROW_RUNTIME( m_nextCycleIndex == 0, + "ParquetDictBasketOutputWriter has non 0 index with no open file" ); } } -void ParquetDictBasketOutputWriter::onFileNameChange( const std::string &fileName ) +void ParquetDictBasketOutputWriter::onFileNameChange( const std::string & fileName ) { ParquetWriter::onFileNameChange( fileName ); - if( m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> length() > 0 ) + if( m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 )->length() > 0 ) { - CSP_TRUE_OR_THROW_RUNTIME( m_indexFileWriterContainer -> isOpen(), "Trying to write basket index data to closed file" ); - m_indexFileWriterContainer -> writeData( { m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) } ); + CSP_TRUE_OR_THROW_RUNTIME( m_indexFileWriterContainer->isOpen(), + "Trying to write basket index data to closed file" ); + m_indexFileWriterContainer->writeData( { m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 ) } ); } - if (m_indexFileWriterContainer->isOpen()) + if( m_indexFileWriterContainer->isOpen() ) { m_indexFileWriterContainer->close(); } - if(!fileName.empty()) + if( !fileName.empty() ) { - m_indexFileWriterContainer - -> open( fileName, m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); + m_indexFileWriterContainer->open( fileName, m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); } - } -SingleColumnParquetOutputHandler *ParquetDictBasketOutputWriter::createScalarOutputHandler( CspTypePtr type, const std::string &name ) +SingleColumnParquetOutputHandler * ParquetDictBasketOutputWriter::createScalarOutputHandler( CspTypePtr type, + const std::string & name ) { m_allHandlers.push_back( std::make_unique( m_engine, *this, type, name ) ); - return static_cast(m_allHandlers.back().get()); + return static_cast( m_allHandlers.back().get() ); } -StructParquetOutputHandler *ParquetDictBasketOutputWriter::createStructOutputHandler( CspTypePtr type, - const DictionaryPtr &fieldMap ) +StructParquetOutputHandler * ParquetDictBasketOutputWriter::createStructOutputHandler( CspTypePtr type, + const DictionaryPtr & fieldMap ) { m_allHandlers.push_back( std::make_unique( m_engine, *this, type, fieldMap ) ); - return static_cast(m_allHandlers.back().get()); + return static_cast( m_allHandlers.back().get() ); } - -ParquetScalarDictBasketOutputWriter::ParquetScalarDictBasketOutputWriter( ParquetOutputAdapterManager *outputAdapterManager, - const std::string &columnName, - CspTypePtr cspTypePtr ) - : ParquetDictBasketOutputWriter( outputAdapterManager, columnName ) +ParquetScalarDictBasketOutputWriter::ParquetScalarDictBasketOutputWriter( + ParquetOutputAdapterManager * outputAdapterManager, const std::string & columnName, CspTypePtr cspTypePtr ) + : ParquetDictBasketOutputWriter( outputAdapterManager, columnName ) { m_valueOutputAdapter = getScalarOutputHandler( cspTypePtr, columnName ); } -void ParquetScalarDictBasketOutputWriter::writeValue( const std::string &valueKey, const TimeSeriesProvider *ts ) +void ParquetScalarDictBasketOutputWriter::writeValue( const std::string & valueKey, const TimeSeriesProvider * ts ) { - m_valueOutputAdapter -> writeValueFromTs( ts ); - ParquetDictBasketOutputWriter::writeValue(valueKey, ts); + m_valueOutputAdapter->writeValueFromTs( ts ); + ParquetDictBasketOutputWriter::writeValue( valueKey, ts ); } -ParquetStructDictBasketOutputWriter::ParquetStructDictBasketOutputWriter( ParquetOutputAdapterManager *outputAdapterManager, - const std::string &columnName, - CspTypePtr cspTypePtr ) - : ParquetDictBasketOutputWriter( outputAdapterManager, columnName ) +ParquetStructDictBasketOutputWriter::ParquetStructDictBasketOutputWriter( + ParquetOutputAdapterManager * outputAdapterManager, const std::string & columnName, CspTypePtr cspTypePtr ) + : ParquetDictBasketOutputWriter( outputAdapterManager, columnName ) { // We don't support fieldMap for now, only default field map - auto structMetaPtr = std::static_pointer_cast( cspTypePtr ) -> meta().get(); + auto structMetaPtr = std::static_pointer_cast( cspTypePtr )->meta().get(); DictionaryPtr dict = std::make_shared(); - for(auto&& field:structMetaPtr->fields()) + for( auto && field : structMetaPtr->fields() ) { - dict->insert(field->fieldname(), columnName + "." + field->fieldname()); + dict->insert( field->fieldname(), columnName + "." + field->fieldname() ); } m_valueOutputAdapter = getStructOutputHandler( cspTypePtr, dict ); } -void ParquetStructDictBasketOutputWriter::writeValue( const std::string &valueKey, const TimeSeriesProvider *ts ) +void ParquetStructDictBasketOutputWriter::writeValue( const std::string & valueKey, const TimeSeriesProvider * ts ) { - m_valueOutputAdapter -> writeValueFromTs( ts ); - ParquetDictBasketOutputWriter::writeValue(valueKey, ts); + m_valueOutputAdapter->writeValueFromTs( ts ); + ParquetDictBasketOutputWriter::writeValue( valueKey, ts ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.cpp b/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.cpp index 3e299260..d0caea06 100644 --- a/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.cpp +++ b/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.cpp @@ -1,24 +1,21 @@ +#include #include #include #include -#include namespace csp::adapters::parquet { -ParquetFileReaderWrapper::~ParquetFileReaderWrapper() -{ - close(); -} +ParquetFileReaderWrapper::~ParquetFileReaderWrapper() { close(); } -void ParquetFileReaderWrapper::open( const std::string &fileName ) +void ParquetFileReaderWrapper::open( const std::string & fileName ) { FileReaderWrapper::open( fileName ); try { STATUS_OK_OR_THROW_RUNTIME( - ::parquet::arrow::OpenFile( m_inputFile, arrow::default_memory_pool(), &m_fileReader ), - "Failed to open parquet file " << fileName ); + ::parquet::arrow::OpenFile( m_inputFile, arrow::default_memory_pool(), &m_fileReader ), + "Failed to open parquet file " << fileName ); } catch( ... ) { @@ -34,11 +31,12 @@ void ParquetFileReaderWrapper::close() FileReaderWrapper::close(); } -bool ParquetFileReaderWrapper::readNextRowGroup( const std::vector neededColumns, std::shared_ptr<::arrow::Table> &dst ) +bool ParquetFileReaderWrapper::readNextRowGroup( const std::vector neededColumns, + std::shared_ptr<::arrow::Table> & dst ) { - if( m_fileReader -> num_row_groups() > m_nextRowGroup ) + if( m_fileReader->num_row_groups() > m_nextRowGroup ) { - STATUS_OK_OR_THROW_RUNTIME( m_fileReader -> ReadRowGroup( m_nextRowGroup, neededColumns, &dst ), + STATUS_OK_OR_THROW_RUNTIME( m_fileReader->ReadRowGroup( m_nextRowGroup, neededColumns, &dst ), "Failed to read row group " << m_nextRowGroup << " from file " << m_fileName ); ++m_nextRowGroup; return true; @@ -47,14 +45,12 @@ bool ParquetFileReaderWrapper::readNextRowGroup( const std::vector neededCo { dst = nullptr; return false; - } } -void ParquetFileReaderWrapper::getSchema( std::shared_ptr<::arrow::Schema> &dst ) +void ParquetFileReaderWrapper::getSchema( std::shared_ptr<::arrow::Schema> & dst ) { - STATUS_OK_OR_THROW_RUNTIME( m_fileReader -> GetSchema( &dst ), - "Failed to get schema from file " << m_fileName ); + STATUS_OK_OR_THROW_RUNTIME( m_fileReader->GetSchema( &dst ), "Failed to get schema from file " << m_fileName ); } -} \ No newline at end of file +} // namespace csp::adapters::parquet \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp b/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp index badd963a..7fa6fd89 100644 --- a/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp +++ b/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp @@ -1,30 +1,28 @@ -#include -#include #include #include +#include +#include #include namespace csp::adapters::parquet { -void ParquetFileWriterWrapper::openImpl( const std::string &fileName, const std::string &compression ) +void ParquetFileWriterWrapper::openImpl( const std::string & fileName, const std::string & compression ) { CSP_TRUE_OR_THROW_RUNTIME( m_outputStream == nullptr, "Trying to open parquet file while previous was not closed" ); - PARQUET_ASSIGN_OR_THROW( - m_outputStream, - arrow::io::FileOutputStream::Open( fileName.c_str())); + PARQUET_ASSIGN_OR_THROW( m_outputStream, arrow::io::FileOutputStream::Open( fileName.c_str() ) ); ::parquet::WriterProperties::Builder builder; - builder.compression( resolveCompression( compression )); - builder.version(::parquet::ParquetVersion::PARQUET_2_0 ); + builder.compression( resolveCompression( compression ) ); + builder.version( ::parquet::ParquetVersion::PARQUET_2_0 ); ::parquet::ArrowWriterProperties::Builder arrowBuilder; arrowBuilder.store_schema(); - STATUS_OK_OR_THROW_RUNTIME( - ::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), m_outputStream, builder.build(), arrowBuilder.build(), - &m_fileWriter ), - "Failed to open parquet file writer" ); + STATUS_OK_OR_THROW_RUNTIME( ::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), + m_outputStream, builder.build(), + arrowBuilder.build(), &m_fileWriter ), + "Failed to open parquet file writer" ); } void ParquetFileWriterWrapper::close() @@ -33,22 +31,23 @@ void ParquetFileWriterWrapper::close() { // Let's move them first, if there are any exceptions, we still want the pointer to be null std::shared_ptr<::arrow::io::FileOutputStream> outputStream{ std::move( m_outputStream ) }; - std::unique_ptr<::parquet::arrow::FileWriter> fileWriter{ std::move( m_fileWriter ) }; + std::unique_ptr<::parquet::arrow::FileWriter> fileWriter{ std::move( m_fileWriter ) }; // Should be done by move constructor but let's be safe: m_outputStream = nullptr; m_fileWriter = nullptr; - if(fileWriter) + if( fileWriter ) STATUS_OK_OR_THROW_RUNTIME( fileWriter->Close(), "Failed to close parquet file writer" ); - if(outputStream) + if( outputStream ) STATUS_OK_OR_THROW_RUNTIME( outputStream->Close(), "Failed to close parquet output stream" ); } } -void ParquetFileWriterWrapper::writeTable( const std::shared_ptr<::arrow::Table> &table ) +void ParquetFileWriterWrapper::writeTable( const std::shared_ptr<::arrow::Table> & table ) { - CSP_TRUE_OR_THROW_RUNTIME(m_fileWriter, "File writer is null!!!"); - STATUS_OK_OR_THROW_RUNTIME( m_fileWriter->WriteTable( *table, table->num_rows()), "Failed to write to parquet file" ); + CSP_TRUE_OR_THROW_RUNTIME( m_fileWriter, "File writer is null!!!" ); + STATUS_OK_OR_THROW_RUNTIME( m_fileWriter->WriteTable( *table, table->num_rows() ), + "Failed to write to parquet file" ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetInputAdapterManager.cpp b/cpp/csp/adapters/parquet/ParquetInputAdapterManager.cpp index 4109830a..d8bd3a94 100644 --- a/cpp/csp/adapters/parquet/ParquetInputAdapterManager.cpp +++ b/cpp/csp/adapters/parquet/ParquetInputAdapterManager.cpp @@ -1,32 +1,34 @@ +#include #include -#include #include #include -#include +#include namespace csp::adapters::parquet { -ParquetInputAdapterManager::ParquetInputAdapterManager( csp::Engine *engine, const Dictionary &properties, - GeneratorPtr generatorPtr, - TableGeneratorPtr tableGeneratorPtr) : - AdapterManager( engine ), - m_fileNameGeneratorReplicator( generatorPtr ? std::make_shared( generatorPtr ) : nullptr ), - m_time_shift(0, 0), - m_tableGenerator(tableGeneratorPtr), - m_reader() +ParquetInputAdapterManager::ParquetInputAdapterManager( csp::Engine * engine, const Dictionary & properties, + GeneratorPtr generatorPtr, TableGeneratorPtr tableGeneratorPtr ) + : AdapterManager( engine ) + , m_fileNameGeneratorReplicator( generatorPtr ? std::make_shared( generatorPtr ) + : nullptr ) + , m_time_shift( 0, 0 ) + , m_tableGenerator( tableGeneratorPtr ) + , m_reader() { - CSP_TRUE_OR_THROW_RUNTIME(!generatorPtr || !m_tableGenerator, "Trying to set both generatorPtr and tableGeneratorPtr"); - CSP_TRUE_OR_THROW_RUNTIME(generatorPtr || m_tableGenerator, "Either generatorPtr or tableGeneratorPtr must be set"); - - m_symbolColumn = properties.get( "symbol_column", "" ); - m_timeColumn = properties.get( "time_column", "" ); - m_defaultTimezone = properties.get( "tz", "UTC" ); - m_splitColumnsToFiles = properties.get( "split_columns_to_files" ); - m_isArrowIPC = properties.get( "is_arrow_ipc", false ); + CSP_TRUE_OR_THROW_RUNTIME( !generatorPtr || !m_tableGenerator, + "Trying to set both generatorPtr and tableGeneratorPtr" ); + CSP_TRUE_OR_THROW_RUNTIME( generatorPtr || m_tableGenerator, + "Either generatorPtr or tableGeneratorPtr must be set" ); + + m_symbolColumn = properties.get( "symbol_column", "" ); + m_timeColumn = properties.get( "time_column", "" ); + m_defaultTimezone = properties.get( "tz", "UTC" ); + m_splitColumnsToFiles = properties.get( "split_columns_to_files" ); + m_isArrowIPC = properties.get( "is_arrow_ipc", false ); m_allowOverlappingPeriods = properties.get( "allow_overlapping_periods", false ); - m_allowMissingColumns = properties.get( "allow_missing_columns", false ); - m_allowMissingFiles = properties.get( "allow_missing_files", false ); + m_allowMissingColumns = properties.get( "allow_missing_columns", false ); + m_allowMissingFiles = properties.get( "allow_missing_files", false ); properties.tryGet( "start_time", m_startTime ); properties.tryGet( "end_time", m_endTime ); properties.tryGet( "time_shift", m_time_shift ); @@ -36,29 +38,27 @@ ParquetInputAdapterManager::ParquetInputAdapterManager( csp::Engine *engine, con "Only UTC default timezone is supported, got:" << m_defaultTimezone ); } -ParquetInputAdapterManager::~ParquetInputAdapterManager() -{ -} +ParquetInputAdapterManager::~ParquetInputAdapterManager() {} void ParquetInputAdapterManager::start( DateTime starttime, DateTime endtime ) { if( !m_startTime.isNone() ) { - starttime = std::max(starttime, m_startTime); + starttime = std::max( starttime, m_startTime ); } AdapterManager::start( starttime, endtime ); CSP_TRUE_OR_THROW_RUNTIME( m_reader == nullptr, "Starting parquet adapter manager more than once" ); - if(m_fileNameGeneratorReplicator) + if( m_fileNameGeneratorReplicator ) { - m_fileNameGeneratorReplicator -> init( starttime, endtime ); + m_fileNameGeneratorReplicator->init( starttime, endtime ); } else { - m_tableGenerator->init(starttime, endtime); + m_tableGenerator->init( starttime, endtime ); } std::optional symbolColumn; - std::set neededColumns; + std::set neededColumns; if( m_symbolColumn != "" ) { @@ -67,126 +67,126 @@ void ParquetInputAdapterManager::start( DateTime starttime, DateTime endtime ) } neededColumns.insert( m_timeColumn ); - for( auto &&it : m_dictBasketInputAdapters ) + for( auto && it : m_dictBasketInputAdapters ) { neededColumns.insert( it.first + "__csp_value_count" ); } m_reader = initializeParquetReader( symbolColumn, neededColumns, m_simInputAdapters, true, true ); - if(m_reader == nullptr) + if( m_reader == nullptr ) { return; } - for( auto &&it : m_dictBasketInputAdapters ) + for( auto && it : m_dictBasketInputAdapters ) { - auto valueCountColumnName = it.first + "__csp_value_count"; + auto valueCountColumnName = it.first + "__csp_value_count"; std::string basketSymbolColumn = it.first + +"__csp_symbol"; - DictBasketReaderRecord record{ ( *m_reader )[ valueCountColumnName ], nullptr }; - record.m_valueCountColumn -> ensureType( CspType::UINT16() ); + DictBasketReaderRecord record{ ( *m_reader )[valueCountColumnName], nullptr }; + record.m_valueCountColumn->ensureType( CspType::UINT16() ); neededColumns.insert( valueCountColumnName ); - record.m_reader = initializeParquetReader( basketSymbolColumn, { basketSymbolColumn }, it.second, - false ); + record.m_reader = initializeParquetReader( basketSymbolColumn, { basketSymbolColumn }, it.second, false ); m_dictBasketReaders.push_back( std::move( record ) ); } - m_timestampColumnAdapter = ( *m_reader )[ m_timeColumn ]; + m_timestampColumnAdapter = ( *m_reader )[m_timeColumn]; CSP_TRUE_OR_THROW_RUNTIME( m_timestampColumnAdapter.valid(), "m_timestampColumnAdapter is NULL" ); - m_timestampColumnAdapter -> ensureType( CspType::DATETIME() ); - + m_timestampColumnAdapter->ensureType( CspType::DATETIME() ); } -std::unique_ptr ParquetInputAdapterManager::initializeParquetReader( const std::optional &symbolColumn, - const std::set &neededColumns, - const ParquetInputAdapterManager::AdaptersBySymbol &adaptersBySymbol, - bool subscribeAllOnEmptySymbol, - bool nullOnEmpty) const +std::unique_ptr ParquetInputAdapterManager::initializeParquetReader( + const std::optional & symbolColumn, const std::set & neededColumns, + const ParquetInputAdapterManager::AdaptersBySymbol & adaptersBySymbol, bool subscribeAllOnEmptySymbol, + bool nullOnEmpty ) const { std::set neededColumnsCopy{ neededColumns }; - for( auto &&adaptersForSymbol : adaptersBySymbol ) + for( auto && adaptersForSymbol : adaptersBySymbol ) { - for( auto &&columnAdapterEntryIt : adaptersForSymbol.second.m_adaptersByColumnName ) + for( auto && columnAdapterEntryIt : adaptersForSymbol.second.m_adaptersByColumnName ) { - neededColumnsCopy.insert(columnAdapterEntryIt.first); + neededColumnsCopy.insert( columnAdapterEntryIt.first ); } - for( auto &&structAdapterEntryIt : adaptersForSymbol.second.m_structAdapters ) + for( auto && structAdapterEntryIt : adaptersForSymbol.second.m_structAdapters ) { - auto &&fieldMap = structAdapterEntryIt.first.fieldMap(); - for( auto it = fieldMap -> begin(); it != fieldMap -> end(); ++it ) + auto && fieldMap = structAdapterEntryIt.first.fieldMap(); + for( auto it = fieldMap->begin(); it != fieldMap->end(); ++it ) { neededColumnsCopy.insert( it.key() ); } } } - std::vector columns( neededColumnsCopy.begin(), neededColumnsCopy.end() ); + std::vector columns( neededColumnsCopy.begin(), neededColumnsCopy.end() ); std::unique_ptr reader; if( m_splitColumnsToFiles ) { - CSP_TRUE_OR_THROW_RUNTIME(m_fileNameGeneratorReplicator, "Trying to read split columns from file while reading in memory tables"); - reader.reset( new MultipleFileParquetReader( m_fileNameGeneratorReplicator, columns, m_isArrowIPC, m_allowMissingColumns, symbolColumn ) ); + CSP_TRUE_OR_THROW_RUNTIME( m_fileNameGeneratorReplicator, + "Trying to read split columns from file while reading in memory tables" ); + reader.reset( new MultipleFileParquetReader( m_fileNameGeneratorReplicator, columns, m_isArrowIPC, + m_allowMissingColumns, symbolColumn ) ); } else { - if(m_fileNameGeneratorReplicator) + if( m_fileNameGeneratorReplicator ) { - reader.reset( new SingleFileParquetReader( m_fileNameGeneratorReplicator -> getGeneratorReplica(), columns, m_isArrowIPC, - m_allowMissingColumns, m_allowMissingFiles, symbolColumn ) ); + reader.reset( new SingleFileParquetReader( m_fileNameGeneratorReplicator->getGeneratorReplica(), columns, + m_isArrowIPC, m_allowMissingColumns, m_allowMissingFiles, + symbolColumn ) ); } else { - reader.reset( new InMemoryTableParquetReader( m_tableGenerator, columns, m_allowMissingColumns, symbolColumn ) ); + reader.reset( + new InMemoryTableParquetReader( m_tableGenerator, columns, m_allowMissingColumns, symbolColumn ) ); } } - if(!reader->isEmpty()) + if( !reader->isEmpty() ) { - for( auto &symMapPair : adaptersBySymbol ) + for( auto & symMapPair : adaptersBySymbol ) { std::optional symbol; - if( !subscribeAllOnEmptySymbol || - !std::holds_alternative( symMapPair.first ) || - ( std::holds_alternative( symMapPair.first ) && - std::get( symMapPair.first ) != "" ) ) + if( !subscribeAllOnEmptySymbol || !std::holds_alternative( symMapPair.first ) + || ( std::holds_alternative( symMapPair.first ) + && std::get( symMapPair.first ) != "" ) ) { symbol = symMapPair.first; } - for( auto &columnAdapterPair:symMapPair.second.m_adaptersByColumnName ) + for( auto & columnAdapterPair : symMapPair.second.m_adaptersByColumnName ) { - auto &&columnAdapter = ( *reader )[ columnAdapterPair.first ]; - auto &&listReaderInterface = columnAdapterPair.second.m_listReaderInterface; + auto && columnAdapter = ( *reader )[columnAdapterPair.first]; + auto && listReaderInterface = columnAdapterPair.second.m_listReaderInterface; CSP_TRUE_OR_THROW_RUNTIME( !columnAdapter->isListType() || listReaderInterface != nullptr, "Column " << columnAdapterPair.first << " is a list column in parquet file " - << reader -> getCurFileOrTableName() << " while subscribing as non list" ); - CSP_TRUE_OR_THROW_RUNTIME( columnAdapter->isListType() || listReaderInterface == nullptr, - "Column " << columnAdapterPair.first << " is a list is non list in parquet file " - << reader -> getCurFileOrTableName() << " while subscribing to it as list" ); - - if(columnAdapter->isListType()) + << reader->getCurFileOrTableName() + << " while subscribing as non list" ); + CSP_TRUE_OR_THROW_RUNTIME( + columnAdapter->isListType() || listReaderInterface == nullptr, + "Column " << columnAdapterPair.first << " is a list is non list in parquet file " + << reader->getCurFileOrTableName() << " while subscribing to it as list" ); + + if( columnAdapter->isListType() ) { - reader->addListSubscriber(columnAdapterPair.first, columnAdapterPair.second.m_adapter, - symbol, listReaderInterface); - + reader->addListSubscriber( columnAdapterPair.first, columnAdapterPair.second.m_adapter, symbol, + listReaderInterface ); } else { - reader -> addSubscriber( columnAdapterPair.first, columnAdapterPair.second.m_adapter, - symbol ); + reader->addSubscriber( columnAdapterPair.first, columnAdapterPair.second.m_adapter, symbol ); } } - for( auto &structAdapterPair:symMapPair.second.m_structAdapters ) + for( auto & structAdapterPair : symMapPair.second.m_structAdapters ) { CSP_TRUE_OR_THROW_RUNTIME( structAdapterPair.second.m_listReaderInterface == nullptr, "Struct adapter is not expected to have list reader interface set" ); - reader -> getStructAdapter( structAdapterPair.first ).addSubscriber( - structAdapterPair.second.m_adapter, symbol ); + reader->getStructAdapter( structAdapterPair.first ) + .addSubscriber( structAdapterPair.second.m_adapter, symbol ); } } } - if(!reader->start() && nullOnEmpty) + if( !reader->start() && nullOnEmpty ) { - return std::unique_ptr(nullptr); + return std::unique_ptr( nullptr ); } return reader; } @@ -197,33 +197,34 @@ void ParquetInputAdapterManager::stop() m_dictBasketReaders.clear(); m_timestampColumnAdapter = nullptr; m_fileNameGeneratorReplicator = nullptr; - m_tableGenerator = nullptr; + m_tableGenerator = nullptr; AdapterManager::stop(); } DateTime ParquetInputAdapterManager::processNextSimTimeSlice( DateTime time ) { - if( unlikely( !m_reader || !m_reader -> hasData() ) ) + if( unlikely( !m_reader || !m_reader->hasData() ) ) { return DateTime::NONE(); } auto data_reference_time = time - m_time_shift; - auto nextDataTime = m_timestampColumnAdapter -> getCurValue(); + auto nextDataTime = m_timestampColumnAdapter->getCurValue(); while( !nextDataTime.value().isNone() && nextDataTime.value() < data_reference_time ) { - for( auto &&dictBasketRecord:m_dictBasketReaders ) + for( auto && dictBasketRecord : m_dictBasketReaders ) { - auto numValuesToSkip = dictBasketRecord.m_valueCountColumn -> getCurValue().value(); - dictBasketRecord.m_reader -> skipRows( numValuesToSkip ); + auto numValuesToSkip = dictBasketRecord.m_valueCountColumn->getCurValue().value(); + dictBasketRecord.m_reader->skipRows( numValuesToSkip ); } - if(!m_reader -> skipRow()) + if( !m_reader->skipRow() ) { nextDataTime = DateTime::NONE(); break; } - nextDataTime = m_timestampColumnAdapter -> getCurValue(); + nextDataTime = m_timestampColumnAdapter->getCurValue(); } - if( unlikely( nextDataTime.value().isNone() || ( !m_endTime.isNone() && ( m_endTime - m_time_shift ) < nextDataTime ) ) ) + if( unlikely( nextDataTime.value().isNone() + || ( !m_endTime.isNone() && ( m_endTime - m_time_shift ) < nextDataTime ) ) ) { return DateTime::NONE(); } @@ -233,43 +234,44 @@ DateTime ParquetInputAdapterManager::processNextSimTimeSlice( DateTime time ) return nextDataTime.value() + m_time_shift; } - CSP_TRUE_OR_THROW_RUNTIME( data_reference_time == nextDataTime, "Expected time " << nextDataTime.value() << " got " << data_reference_time ); + CSP_TRUE_OR_THROW_RUNTIME( data_reference_time == nextDataTime, + "Expected time " << nextDataTime.value() << " got " << data_reference_time ); do { - for( auto &&dictBasketRecord:m_dictBasketReaders ) + for( auto && dictBasketRecord : m_dictBasketReaders ) { - auto numValuesToDispatch = dictBasketRecord.m_valueCountColumn -> getCurValue().value(); + auto numValuesToDispatch = dictBasketRecord.m_valueCountColumn->getCurValue().value(); for( uint16_t i = 0; i < numValuesToDispatch; ++i ) { - dictBasketRecord.m_reader -> dispatchRow(); + dictBasketRecord.m_reader->dispatchRow(); } } - m_reader -> dispatchRow(); + m_reader->dispatchRow(); - nextDataTime = m_reader -> hasData() ? m_timestampColumnAdapter -> getCurValue() : DateTime::NONE(); + nextDataTime = m_reader->hasData() ? m_timestampColumnAdapter->getCurValue() : DateTime::NONE(); } while( !nextDataTime.value().isNone() && nextDataTime == data_reference_time ); - if( unlikely( nextDataTime -> isNone() ) ) + if( unlikely( nextDataTime->isNone() ) ) { return DateTime::NONE(); } - if(m_allowOverlappingPeriods && nextDataTime.value() < data_reference_time) + if( m_allowOverlappingPeriods && nextDataTime.value() < data_reference_time ) { return time + TimeDelta( 0, 1 ); } return nextDataTime.value() + m_time_shift; } - -ManagedSimInputAdapter *ParquetInputAdapterManager::getInputAdapter( CspTypePtr &type, const Dictionary &properties, PushMode pushMode, - const DialectGenericListReaderInterface::Ptr &listReaderInterface ) +ManagedSimInputAdapter * +ParquetInputAdapterManager::getInputAdapter( CspTypePtr & type, const Dictionary & properties, PushMode pushMode, + const DialectGenericListReaderInterface::Ptr & listReaderInterface ) { CSP_TRUE_OR_THROW( !m_pushMode.has_value() || m_pushMode.value() == pushMode, NotImplemented, - "Subscribing with varying push modes is not currently supported. previous=" << m_pushMode.value() - << " current=" << pushMode ); - m_pushMode = pushMode; + "Subscribing with varying push modes is not currently supported. previous=" + << m_pushMode.value() << " current=" << pushMode ); + m_pushMode = pushMode; std::string basketName = properties.get( "basket_name", "" ); utils::Symbol symbol = ""; @@ -290,19 +292,20 @@ ManagedSimInputAdapter *ParquetInputAdapterManager::getInputAdapter( CspTypePtr } else { - CSP_TRUE_OR_THROW(listReaderInterface == nullptr, NotImplemented, "Reading of baskets of arrays is unsupported"); + CSP_TRUE_OR_THROW( listReaderInterface == nullptr, NotImplemented, + "Reading of baskets of arrays is unsupported" ); return getDictBasketAdapter( type, properties, pushMode, symbol, basketName ); } } ManagedSimInputAdapter * -ParquetInputAdapterManager::getRegularAdapter( const CspTypePtr &type, const Dictionary &properties, const PushMode &pushMode, - const utils::Symbol &symbol, - const DialectGenericListReaderInterface::Ptr &listReaderInterface ) +ParquetInputAdapterManager::getRegularAdapter( const CspTypePtr & type, const Dictionary & properties, + const PushMode & pushMode, const utils::Symbol & symbol, + const DialectGenericListReaderInterface::Ptr & listReaderInterface ) { if( pushMode == PushMode::NON_COLLAPSING ) { - if( std::holds_alternative( symbol) && std::get( symbol ).empty() ) + if( std::holds_alternative( symbol ) && std::get( symbol ).empty() ) { m_subscribedForAll = true; } @@ -310,16 +313,15 @@ ParquetInputAdapterManager::getRegularAdapter( const CspTypePtr &type, const Dic { m_subscribedBySymbol = true; } - // If we subscribe both by symbol and subscribe all, we might have weird issues. Consider the following scenario: - // ts1: subscribe AAPL - // ts2: subscribe IBM - // ts3: subscribe_all - // Assume that the file has 2 entries, both on the same timestamp, one for AAPL, one for IBM. - // Then ts1 and ts2 will output on the first cycle the AAPL and IBM data, ts3 will output data for AAPL only, and on next - // cycle will output the data for IBM. This behavior of same data ticking on different cycles could create a lot of issues - // and generally buggy so we want to avoid it and forbid subscribing both ways. - CSP_TRUE_OR_THROW( !m_subscribedBySymbol || !m_subscribedForAll, NotImplemented, - "Subscribing both by symbol and without symbol for same parquet reader is not currently supported" ); + // If we subscribe both by symbol and subscribe all, we might have weird issues. Consider the following + // scenario: ts1: subscribe AAPL ts2: subscribe IBM ts3: subscribe_all Assume that the file has 2 entries, both + // on the same timestamp, one for AAPL, one for IBM. Then ts1 and ts2 will output on the first cycle the AAPL + // and IBM data, ts3 will output data for AAPL only, and on next cycle will output the data for IBM. This + // behavior of same data ticking on different cycles could create a lot of issues and generally buggy so we want + // to avoid it and forbid subscribing both ways. + CSP_TRUE_OR_THROW( + !m_subscribedBySymbol || !m_subscribedForAll, NotImplemented, + "Subscribing both by symbol and without symbol for same parquet reader is not currently supported" ); } auto fieldMap = properties.getUntypedValue( "field_map" ); @@ -330,7 +332,8 @@ ParquetInputAdapterManager::getRegularAdapter( const CspTypePtr &type, const Dic } else if( std::holds_alternative( fieldMap ) ) { - CSP_TRUE_OR_THROW(listReaderInterface == nullptr, NotImplemented, "Reading of arrays of structs is unsupported"); + CSP_TRUE_OR_THROW( listReaderInterface == nullptr, NotImplemented, + "Reading of arrays of structs is unsupported" ); auto dictFieldMap = properties.get( "field_map" ); return getStructAdapter( type, symbol, dictFieldMap, pushMode ); } @@ -342,14 +345,15 @@ ParquetInputAdapterManager::getRegularAdapter( const CspTypePtr &type, const Dic CSP_THROW( RuntimeException, "Reached unreachable code" ); } -ManagedSimInputAdapter *ParquetInputAdapterManager::getDictBasketAdapter( const CspTypePtr &type, - const Dictionary &properties, const PushMode &pushMode, - const utils::Symbol &symbol, - const std::string &basketName ) +ManagedSimInputAdapter * ParquetInputAdapterManager::getDictBasketAdapter( const CspTypePtr & type, + const Dictionary & properties, + const PushMode & pushMode, + const utils::Symbol & symbol, + const std::string & basketName ) { auto fieldMap = properties.getUntypedValue( "field_map" ); - auto &&basketRecord = m_dictBasketInputAdapters[ basketName ]; + auto && basketRecord = m_dictBasketInputAdapters[basketName]; if( std::holds_alternative( fieldMap ) ) { @@ -370,11 +374,10 @@ ManagedSimInputAdapter *ParquetInputAdapterManager::getDictBasketAdapter( const CSP_THROW( RuntimeException, "Reached unreachable code" ); } -ManagedSimInputAdapter * -ParquetInputAdapterManager::getOrCreateSingleColumnAdapter( ParquetInputAdapterManager::AdaptersBySymbol &inputAdaptersContainer, - const CspTypePtr &type, const utils::Symbol &symbol, const std::string &field, - const PushMode &pushMode, - const DialectGenericListReaderInterface::Ptr &listReaderInterface ) +ManagedSimInputAdapter * ParquetInputAdapterManager::getOrCreateSingleColumnAdapter( + ParquetInputAdapterManager::AdaptersBySymbol & inputAdaptersContainer, const CspTypePtr & type, + const utils::Symbol & symbol, const std::string & field, const PushMode & pushMode, + const DialectGenericListReaderInterface::Ptr & listReaderInterface ) { auto itBySymbol = inputAdaptersContainer.find( symbol ); if( itBySymbol == inputAdaptersContainer.end() ) @@ -382,33 +385,32 @@ ParquetInputAdapterManager::getOrCreateSingleColumnAdapter( ParquetInputAdapterM itBySymbol = inputAdaptersContainer.emplace( symbol, AdaptersSingleSymbol() ).first; } - auto itByColumn = itBySymbol -> second.m_adaptersByColumnName.find( field ); - const CspTypePtr& adapterType = (listReaderInterface==nullptr) ? type : CspType::DIALECT_GENERIC(); + auto itByColumn = itBySymbol->second.m_adaptersByColumnName.find( field ); + const CspTypePtr & adapterType = ( listReaderInterface == nullptr ) ? type : CspType::DIALECT_GENERIC(); - if( itByColumn == itBySymbol -> second.m_adaptersByColumnName.end() ) + if( itByColumn == itBySymbol->second.m_adaptersByColumnName.end() ) { - itByColumn = itBySymbol -> second.m_adaptersByColumnName.emplace( - field, AdapterInfo{ engine() -> createOwnedObject( - adapterType, this, - pushMode ), listReaderInterface } ).first; + itByColumn = itBySymbol->second.m_adaptersByColumnName + .emplace( field, + AdapterInfo{ engine()->createOwnedObject( adapterType, this, + pushMode ), + listReaderInterface } ) + .first; } - return itByColumn -> second.m_adapter; + return itByColumn->second.m_adapter; } ManagedSimInputAdapter * -ParquetInputAdapterManager::getSingleColumnAdapter( const CspTypePtr &type, const utils::Symbol &symbol, - const std::string &field, PushMode pushMode, - const DialectGenericListReaderInterface::Ptr &listReaderInterface) +ParquetInputAdapterManager::getSingleColumnAdapter( const CspTypePtr & type, const utils::Symbol & symbol, + const std::string & field, PushMode pushMode, + const DialectGenericListReaderInterface::Ptr & listReaderInterface ) { - return getOrCreateSingleColumnAdapter( m_simInputAdapters, type, symbol, field, - pushMode, listReaderInterface ); + return getOrCreateSingleColumnAdapter( m_simInputAdapters, type, symbol, field, pushMode, listReaderInterface ); } - -ManagedSimInputAdapter *ParquetInputAdapterManager::getOrCreateStructColumnAdapter( AdaptersBySymbol &inputAdaptersContainer, - const CspTypePtr &type, const utils::Symbol &symbol, - const csp::DictionaryPtr &fieldMap, - const PushMode &pushMode ) +ManagedSimInputAdapter * ParquetInputAdapterManager::getOrCreateStructColumnAdapter( + AdaptersBySymbol & inputAdaptersContainer, const CspTypePtr & type, const utils::Symbol & symbol, + const csp::DictionaryPtr & fieldMap, const PushMode & pushMode ) { auto itBySymbol = inputAdaptersContainer.find( symbol ); if( itBySymbol == inputAdaptersContainer.end() ) @@ -418,25 +420,23 @@ ManagedSimInputAdapter *ParquetInputAdapterManager::getOrCreateStructColumnAdapt StructAdapterInfo key{ type, fieldMap }; - auto itByColumn = itBySymbol -> second.m_structAdapters.find( key ); + auto itByColumn = itBySymbol->second.m_structAdapters.find( key ); - if( itByColumn == itBySymbol -> second.m_structAdapters.end() ) + if( itByColumn == itBySymbol->second.m_structAdapters.end() ) { - itByColumn = itBySymbol -> second.m_structAdapters.emplace( - key, engine() -> createOwnedObject( - type, this, - pushMode ) ).first; + itByColumn = itBySymbol->second.m_structAdapters + .emplace( key, engine()->createOwnedObject( type, this, pushMode ) ) + .first; } - return itByColumn -> second.m_adapter; - + return itByColumn->second.m_adapter; } -ManagedSimInputAdapter * -ParquetInputAdapterManager::getStructAdapter( const CspTypePtr &type, const utils::Symbol &symbol, - const csp::DictionaryPtr &fieldMap, PushMode pushMode ) +ManagedSimInputAdapter * ParquetInputAdapterManager::getStructAdapter( const CspTypePtr & type, + const utils::Symbol & symbol, + const csp::DictionaryPtr & fieldMap, + PushMode pushMode ) { return getOrCreateStructColumnAdapter( m_simInputAdapters, type, symbol, fieldMap, pushMode ); } - -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp index f7309225..430febdd 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp +++ b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp @@ -7,17 +7,13 @@ using namespace csp::adapters::parquet; namespace csp::adapters::parquet { -uint32_t ParquetOutputHandler::getChunkSize() const -{ - return m_parquetWriter.getChunkSize(); -} - +uint32_t ParquetOutputHandler::getChunkSize() const { return m_parquetWriter.getChunkSize(); } -SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, - std::string columnName ) - : ParquetOutputHandler( parquetWriter, type ) +SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine * engine, ParquetWriter & parquetWriter, + CspTypePtr & type, std::string columnName ) + : ParquetOutputHandler( parquetWriter, type ) { - switch( m_type -> type() ) + switch( m_type->type() ) { case CspType::TypeTraits::BOOL: createColumnBuilder( columnName ); @@ -58,13 +54,13 @@ SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine *engi case CspType::TypeTraits::DATE: createColumnBuilder( columnName ); break; - case CspType::TypeTraits::TIME: - createColumnBuilder( columnName ); - break; + case CspType::TypeTraits::TIME: + createColumnBuilder( columnName ); + break; case CspType::TypeTraits::STRING: { - const CspStringType &strType = static_cast(*type); + const CspStringType & strType = static_cast( *type ); if( strType.isBytes() ) { createColumnBuilder( columnName ); @@ -77,39 +73,39 @@ SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine *engi } case CspType::TypeTraits::ENUM: { - auto enumMetaPtr = std::static_pointer_cast( type ) -> meta(); + auto enumMetaPtr = std::static_pointer_cast( type )->meta(); createEnumColumnBuilder( columnName, enumMetaPtr ); break; } default: { - CSP_THROW( TypeError, "Writing of " << m_type -> type().asString() << " to parquet is not supported" ); + CSP_THROW( TypeError, "Writing of " << m_type->type().asString() << " to parquet is not supported" ); } } } - -template< typename ColumnBuilder > -inline void SingleColumnParquetOutputHandler::createColumnBuilder( const std::string &columnName ) +template +inline void SingleColumnParquetOutputHandler::createColumnBuilder( const std::string & columnName ) { m_columnArrayBuilder = std::make_unique( columnName, getChunkSize() ); m_valueHandler = std::make_unique( - [ this ]( const TimeSeriesProvider *input ) - { - static_cast(this -> m_columnArrayBuilder.get()) - -> setValue( input -> lastValueTyped() ); - } ); + [this]( const TimeSeriesProvider * input ) + { + static_cast( this->m_columnArrayBuilder.get() ) + ->setValue( input->lastValueTyped() ); + } ); } -void SingleColumnParquetOutputHandler::createEnumColumnBuilder( const std::string &columnName, CspEnumMeta::Ptr enumMetaPtr ) +void SingleColumnParquetOutputHandler::createEnumColumnBuilder( const std::string & columnName, + CspEnumMeta::Ptr enumMetaPtr ) { m_columnArrayBuilder = std::make_unique( columnName, getChunkSize() ); m_valueHandler = std::make_unique( - [ this ]( const TimeSeriesProvider *input ) - { - static_cast(this -> m_columnArrayBuilder.get()) - -> setValue( input -> lastValueTyped().name() ); - } ); + [this]( const TimeSeriesProvider * input ) + { + static_cast( this->m_columnArrayBuilder.get() ) + ->setValue( input->lastValueTyped().name() ); + } ); } void SingleColumnParquetOutputAdapter::executeImpl() @@ -118,58 +114,62 @@ void SingleColumnParquetOutputAdapter::executeImpl() m_parquetWriter.scheduleEndCycleEvent(); } -ListColumnParquetOutputHandler::ListColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &elemType, - const std::string &columnName, - DialectGenericListWriterInterface::Ptr &listWriterInterface ) - : ParquetOutputHandler( parquetWriter, CspType::DIALECT_GENERIC() ), - m_columnArrayBuilder( - std::make_shared( columnName, getChunkSize(), createValueBuilder( elemType, listWriterInterface ), - listWriterInterface ) ) +ListColumnParquetOutputHandler::ListColumnParquetOutputHandler( + Engine * engine, ParquetWriter & parquetWriter, CspTypePtr & elemType, const std::string & columnName, + DialectGenericListWriterInterface::Ptr & listWriterInterface ) + : ParquetOutputHandler( parquetWriter, CspType::DIALECT_GENERIC() ) + , m_columnArrayBuilder( std::make_shared( + columnName, getChunkSize(), createValueBuilder( elemType, listWriterInterface ), listWriterInterface ) ) { m_valueHandler = std::make_unique( - [ this ]( const TimeSeriesProvider *input ) - { - static_cast(this -> m_columnArrayBuilder.get()) - -> setValue( input -> lastValueTyped() ); - } ); + [this]( const TimeSeriesProvider * input ) + { + static_cast( this->m_columnArrayBuilder.get() ) + ->setValue( input->lastValueTyped() ); + } ); } namespace { -template< typename A, typename V = typename A::value_type > -inline std::shared_ptr<::arrow::ArrayBuilder> makeArrayAndAttachToWriter( DialectGenericListWriterInterface::Ptr &listWriterInterface ) -{ - auto&& typedWriter = std::dynamic_pointer_cast>( listWriterInterface ); - auto& listWriterInterfaceRef = *listWriterInterface; - CSP_TRUE_OR_THROW( typedWriter != nullptr, TypeError, - "Expected " << typeid( TypedDialectGenericListWriterInterface ).name() << " " << " got " << - typeid( listWriterInterfaceRef ).name() ); - - auto res = std::make_shared(); - typedWriter -> setWriteFunction( - [ res ]( const V &value ){ STATUS_OK_OR_THROW_RUNTIME( res -> Append( value ), "Failed to append value to list array" ); } ); - return res; -} + template + inline std::shared_ptr<::arrow::ArrayBuilder> + makeArrayAndAttachToWriter( DialectGenericListWriterInterface::Ptr & listWriterInterface ) + { + auto && typedWriter + = std::dynamic_pointer_cast>( listWriterInterface ); + auto & listWriterInterfaceRef = *listWriterInterface; + CSP_TRUE_OR_THROW( typedWriter != nullptr, TypeError, + "Expected " << typeid( TypedDialectGenericListWriterInterface ).name() << " " << " got " + << typeid( listWriterInterfaceRef ).name() ); + + auto res = std::make_shared(); + typedWriter->setWriteFunction( + [res]( const V & value ) + { STATUS_OK_OR_THROW_RUNTIME( res->Append( value ), "Failed to append value to list array" ); } ); + return res; + } -} +} // namespace -std::shared_ptr<::arrow::ArrayBuilder> ListColumnParquetOutputHandler::createValueBuilder( const CspTypePtr &elemType, - DialectGenericListWriterInterface::Ptr &listWriterInterface ) +std::shared_ptr<::arrow::ArrayBuilder> +ListColumnParquetOutputHandler::createValueBuilder( const CspTypePtr & elemType, + DialectGenericListWriterInterface::Ptr & listWriterInterface ) { - switch( elemType -> type() ) + switch( elemType->type() ) { case CspType::TypeTraits::BOOL: - return makeArrayAndAttachToWriter(listWriterInterface); + return makeArrayAndAttachToWriter( listWriterInterface ); case CspType::TypeTraits::INT64: - return makeArrayAndAttachToWriter(listWriterInterface); + return makeArrayAndAttachToWriter( listWriterInterface ); case CspType::TypeTraits::DOUBLE: - return makeArrayAndAttachToWriter(listWriterInterface); + return makeArrayAndAttachToWriter( listWriterInterface ); case CspType::TypeTraits::STRING: - return makeArrayAndAttachToWriter(listWriterInterface); + return makeArrayAndAttachToWriter( listWriterInterface ); default: { CSP_THROW( TypeError, - "Writing of list with elements of type " << elemType -> type().asString() << " to parquet is not supported" ); + "Writing of list with elements of type " << elemType->type().asString() + << " to parquet is not supported" ); } } } @@ -180,36 +180,35 @@ void ListColumnParquetOutputAdapter::executeImpl() m_parquetWriter.scheduleEndCycleEvent(); } -StructParquetOutputHandler::StructParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, - DictionaryPtr fieldMap ) - : ParquetOutputHandler( parquetWriter, type ) +StructParquetOutputHandler::StructParquetOutputHandler( Engine * engine, ParquetWriter & parquetWriter, + CspTypePtr & type, DictionaryPtr fieldMap ) + : ParquetOutputHandler( parquetWriter, type ) { - auto structMetaPtr = std::static_pointer_cast( type ) -> meta().get(); + auto structMetaPtr = std::static_pointer_cast( type )->meta().get(); - for( auto it = fieldMap -> begin(); it != fieldMap -> end(); ++it ) + for( auto it = fieldMap->begin(); it != fieldMap->end(); ++it ) { createColumnBuilder( structMetaPtr, it.value(), it.key(), nullptr ); } } -void StructParquetOutputHandler::writeValueFromTs( const TimeSeriesProvider *input ) +void StructParquetOutputHandler::writeValueFromTs( const TimeSeriesProvider * input ) { - const Struct *structData = input -> lastValueTyped().get(); + const Struct * structData = input->lastValueTyped().get(); - for( auto &&valueHandler: m_valueHandlers ) + for( auto && valueHandler : m_valueHandlers ) { valueHandler( structData ); } m_parquetWriter.scheduleEndCycleEvent(); } -inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createColumnBuilder( - const StructMeta *structMeta, - const std::string &columnName, const std::string &structFieldName, - const std::string *path ) +inline StructParquetOutputHandler::ColumnBuilderResultType +StructParquetOutputHandler::createColumnBuilder( const StructMeta * structMeta, const std::string & columnName, + const std::string & structFieldName, const std::string * path ) { - auto fieldPtr = structMeta -> field( structFieldName ).get(); - switch( fieldPtr -> type() -> type() ) + auto fieldPtr = structMeta->field( structFieldName ).get(); + switch( fieldPtr->type()->type() ) { case CspType::TypeTraits::BOOL: return createColumnBuilder( fieldPtr, columnName, path ); @@ -246,32 +245,32 @@ inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHa case CspType::TypeTraits::STRUCT: return createStructColumnBuilder( fieldPtr, columnName, path ); default: - CSP_THROW( TypeError, "Writing of column " << columnName << " of type " << fieldPtr -> type() -> type().asString() - << " to parquet is not supported" ); + CSP_THROW( TypeError, + "Writing of column " << columnName << " of type " << fieldPtr->type()->type().asString() + << " to parquet is not supported" ); } } -template< typename ColumnBuilder > -inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createColumnBuilder( - const StructField *field, - const std::string &columnName, - const std::string *path ) +template +inline StructParquetOutputHandler::ColumnBuilderResultType +StructParquetOutputHandler::createColumnBuilder( const StructField * field, const std::string & columnName, + const std::string * path ) { - std::shared_ptr columnBuilderPtr = std::make_shared( resolveFullColumnName( path, columnName ), - getChunkSize() ); + std::shared_ptr columnBuilderPtr + = std::make_shared( resolveFullColumnName( path, columnName ), getChunkSize() ); auto columnBuilderRawPtr = columnBuilderPtr.get(); - using T = typename ColumnBuilder::ValueTypeT; + using T = typename ColumnBuilder::ValueTypeT; - ValueHandler res = [ field, columnBuilderRawPtr ]( const Struct *s ) + ValueHandler res = [field, columnBuilderRawPtr]( const Struct * s ) { - if( field -> isSet( s ) ) + if( field->isSet( s ) ) { - columnBuilderRawPtr -> setValue( field -> value( s ) ); + columnBuilderRawPtr->setValue( field->value( s ) ); } }; - // We need to collect on the top level value handlers and array builders. Value handlers and builders of nested structs are stored - // in the struct field handlers. Path is non null in this case. + // We need to collect on the top level value handlers and array builders. Value handlers and builders of nested + // structs are stored in the struct field handlers. Path is non null in this case. if( !path ) { m_valueHandlers.push_back( res ); @@ -280,20 +279,20 @@ inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHa return { columnBuilderPtr, res }; } -inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createEnumColumnBuilder( - const StructField *field, - const std::string &columnName, - const std::string *path ) +inline StructParquetOutputHandler::ColumnBuilderResultType +StructParquetOutputHandler::createEnumColumnBuilder( const StructField * field, const std::string & columnName, + const std::string * path ) { - auto columnBuilderPtr{ std::make_shared( resolveFullColumnName( path, columnName ), getChunkSize() ) }; + auto columnBuilderPtr{ + std::make_shared( resolveFullColumnName( path, columnName ), getChunkSize() ) }; auto columnBuilderRawPtr{ columnBuilderPtr.get() }; - auto enumMetaPtr = std::static_pointer_cast( field -> type() ) -> meta(); + auto enumMetaPtr = std::static_pointer_cast( field->type() )->meta(); - ValueHandler res = [ field, columnBuilderRawPtr ]( const Struct *s ) + ValueHandler res = [field, columnBuilderRawPtr]( const Struct * s ) { - if( field -> isSet( s ) ) + if( field->isSet( s ) ) { - columnBuilderRawPtr -> setValue( field -> value( s ).name() ); + columnBuilderRawPtr->setValue( field->value( s ).name() ); } }; if( path == nullptr ) @@ -304,53 +303,47 @@ inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHa return { columnBuilderPtr, res }; } -inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createStructColumnBuilder( - const StructField *structField, - const std::string &columnName, - const std::string *path ) +inline StructParquetOutputHandler::ColumnBuilderResultType +StructParquetOutputHandler::createStructColumnBuilder( const StructField * structField, const std::string & columnName, + const std::string * path ) { - std::vector> fields; + std::vector> fields; std::vector childArrayBuilders; - std::vector childFieldSetters; + std::vector childFieldSetters; - auto structFieldMetaPtr = std::static_pointer_cast( structField -> type() ) -> meta().get(); + auto structFieldMetaPtr = std::static_pointer_cast( structField->type() )->meta().get(); - for( auto &subField:structFieldMetaPtr -> fields() ) + for( auto & subField : structFieldMetaPtr->fields() ) { - std::string fieldSubPath = resolveFullColumnName( path, subField -> fieldname() ); + std::string fieldSubPath = resolveFullColumnName( path, subField->fieldname() ); - auto childBuilderRes = createColumnBuilder( structFieldMetaPtr, - subField -> fieldname(), - subField -> fieldname(), - &fieldSubPath ); + auto childBuilderRes + = createColumnBuilder( structFieldMetaPtr, subField->fieldname(), subField->fieldname(), &fieldSubPath ); childArrayBuilders.push_back( childBuilderRes.m_columnBuilder ); childFieldSetters.push_back( childBuilderRes.m_valueHandler ); fields.push_back( - std::make_shared<::arrow::Field>( subField -> fieldname(), childBuilderRes.m_columnBuilder -> getDataType() ) ); + std::make_shared<::arrow::Field>( subField->fieldname(), childBuilderRes.m_columnBuilder->getDataType() ) ); } - auto subFieldValueSetter = [ childFieldSetters ]( const Struct *s ) + auto subFieldValueSetter = [childFieldSetters]( const Struct * s ) { - for( auto &childFieldSetter:childFieldSetters ) + for( auto & childFieldSetter : childFieldSetters ) { childFieldSetter( s ); } }; auto columnBuilderPtr{ std::make_shared( - resolveFullColumnName( path, columnName ), getChunkSize(), - std::make_shared<::arrow::StructType>( fields ), - childArrayBuilders, - std::move( subFieldValueSetter ) - ) }; + resolveFullColumnName( path, columnName ), getChunkSize(), std::make_shared<::arrow::StructType>( fields ), + childArrayBuilders, std::move( subFieldValueSetter ) ) }; auto columnBuilderRawPtr = columnBuilderPtr.get(); - ValueHandler valueHandler = [ structField, columnBuilderRawPtr ]( const Struct *s ) + ValueHandler valueHandler = [structField, columnBuilderRawPtr]( const Struct * s ) { - if( structField -> isSet( s ) ) + if( structField->isSet( s ) ) { - columnBuilderRawPtr -> setValue( structField -> value( s ).get() ); + columnBuilderRawPtr->setValue( structField->value( s ).get() ); } }; if( path == nullptr ) @@ -361,9 +354,6 @@ inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHa return { columnBuilderPtr, valueHandler }; } -void StructParquetOutputAdapter::executeImpl() -{ - writeValueFromTs( input() ); -} +void StructParquetOutputAdapter::executeImpl() { writeValueFromTs( input() ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp index e662d98d..a24032f8 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp +++ b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp @@ -1,16 +1,18 @@ +#include +#include #include #include -#include -#include #include #include namespace csp::adapters::parquet { - -ParquetOutputAdapterManager::ParquetOutputAdapterManager( csp::Engine *engine, const Dictionary &properties, FileVisitorCallback fileVisitor ) : - AdapterManager( engine ), m_fileVisitor( fileVisitor ), m_outputFilenameAdapter( nullptr ) +ParquetOutputAdapterManager::ParquetOutputAdapterManager( csp::Engine * engine, const Dictionary & properties, + FileVisitorCallback fileVisitor ) + : AdapterManager( engine ) + , m_fileVisitor( fileVisitor ) + , m_outputFilenameAdapter( nullptr ) { m_fileName = properties.get( "file_name" ); m_timestampColumnName = properties.get( "timestamp_column_name" ); @@ -22,28 +24,26 @@ ParquetOutputAdapterManager::ParquetOutputAdapterManager( csp::Engine *engine, c m_parquetWriter = std::make_unique( this, properties ); } -ParquetOutputAdapterManager::~ParquetOutputAdapterManager() -{ -} +ParquetOutputAdapterManager::~ParquetOutputAdapterManager() {} void ParquetOutputAdapterManager::start( DateTime starttime, DateTime endtime ) { - m_parquetWriter -> start(); - for( auto &&writer:m_dictBasketWriters ) + m_parquetWriter->start(); + for( auto && writer : m_dictBasketWriters ) { - writer -> start(); + writer->start(); } } void ParquetOutputAdapterManager::stop() { - bool visitFile = m_fileVisitor && m_parquetWriter -> isFileOpen(); - m_parquetWriter -> stop(); + bool visitFile = m_fileVisitor && m_parquetWriter->isFileOpen(); + m_parquetWriter->stop(); m_parquetWriter = nullptr; - for( auto &&writer:m_dictBasketWriters ) + for( auto && writer : m_dictBasketWriters ) { - writer -> stop(); + writer->stop(); } m_dictBasketWriters.clear(); @@ -51,14 +51,11 @@ void ParquetOutputAdapterManager::stop() m_fileVisitor( m_fileName ); } -DateTime ParquetOutputAdapterManager::processNextSimTimeSlice( DateTime time ) -{ - return DateTime::NONE(); -} +DateTime ParquetOutputAdapterManager::processNextSimTimeSlice( DateTime time ) { return DateTime::NONE(); } -OutputAdapter *ParquetOutputAdapterManager::getOutputAdapter( CspTypePtr &type, const Dictionary &properties ) +OutputAdapter * ParquetOutputAdapterManager::getOutputAdapter( CspTypePtr & type, const Dictionary & properties ) { - if( type -> type() == CspType::Type::STRUCT ) + if( type->type() == CspType::Type::STRUCT ) { return getStructOutputAdapter( type, properties ); } @@ -68,46 +65,49 @@ OutputAdapter *ParquetOutputAdapterManager::getOutputAdapter( CspTypePtr &type, } } -OutputAdapter *ParquetOutputAdapterManager::getListOutputAdapter( CspTypePtr &elemType, const Dictionary &properties, - const DialectGenericListWriterInterface::Ptr& listWriterInterface) +OutputAdapter * +ParquetOutputAdapterManager::getListOutputAdapter( CspTypePtr & elemType, const Dictionary & properties, + const DialectGenericListWriterInterface::Ptr & listWriterInterface ) { auto columnName = properties.get( "column_name" ); - return m_parquetWriter -> getListOutputAdapter( elemType, columnName, listWriterInterface ); + return m_parquetWriter->getListOutputAdapter( elemType, columnName, listWriterInterface ); } - ParquetDictBasketOutputWriter * -ParquetOutputAdapterManager::createDictOutputBasketWriter( const char *columnName, const CspTypePtr &cspTypePtr ) +ParquetOutputAdapterManager::createDictOutputBasketWriter( const char * columnName, const CspTypePtr & cspTypePtr ) { - auto &&existingAdapterIt = m_dictBasketWriterIndexByName.find( columnName ); + auto && existingAdapterIt = m_dictBasketWriterIndexByName.find( columnName ); CSP_TRUE_OR_THROW_RUNTIME( existingAdapterIt == m_dictBasketWriterIndexByName.end(), "Trying to create output basket writer for " << columnName << " more than once" ); - if( cspTypePtr -> type() == CspType::Type::STRUCT ) + if( cspTypePtr->type() == CspType::Type::STRUCT ) { - m_dictBasketWriters.push_back( std::make_unique( this, columnName, cspTypePtr ) ); + m_dictBasketWriters.push_back( + std::make_unique( this, columnName, cspTypePtr ) ); } else { - m_dictBasketWriters.push_back( std::make_unique( this, columnName, cspTypePtr ) ); + m_dictBasketWriters.push_back( + std::make_unique( this, columnName, cspTypePtr ) ); } - m_dictBasketWriterIndexByName[ columnName ] = m_dictBasketWriters.size() - 1; + m_dictBasketWriterIndexByName[columnName] = m_dictBasketWriters.size() - 1; return m_dictBasketWriters.back().get(); } -OutputAdapter *ParquetOutputAdapterManager::createOutputFileNameAdapter() +OutputAdapter * ParquetOutputAdapterManager::createOutputFileNameAdapter() { - CSP_TRUE_OR_THROW_RUNTIME( m_outputFilenameAdapter == nullptr, "Trying to set output filename adapter more than once" ); - m_outputFilenameAdapter = engine() -> createOwnedObject( *this ); + CSP_TRUE_OR_THROW_RUNTIME( m_outputFilenameAdapter == nullptr, + "Trying to set output filename adapter more than once" ); + m_outputFilenameAdapter = engine()->createOwnedObject( *this ); return m_outputFilenameAdapter; } -void ParquetOutputAdapterManager::changeFileName( const std::string &filename ) +void ParquetOutputAdapterManager::changeFileName( const std::string & filename ) { if( m_parquetWriter ) { - m_parquetWriter -> onFileNameChange( filename ); + m_parquetWriter->onFileNameChange( filename ); } if( m_fileVisitor ) @@ -118,28 +118,27 @@ void ParquetOutputAdapterManager::changeFileName( const std::string &filename ) void ParquetOutputAdapterManager::scheduleEndCycle() { - if( rootEngine() -> scheduleEndCycleListener( m_parquetWriter.get() ) ) + if( rootEngine()->scheduleEndCycleListener( m_parquetWriter.get() ) ) { - for( auto &&basketWriter:m_dictBasketWriters ) + for( auto && basketWriter : m_dictBasketWriters ) { - rootEngine() -> scheduleEndCycleListener( basketWriter.get() ); + rootEngine()->scheduleEndCycleListener( basketWriter.get() ); } } } -OutputAdapter *ParquetOutputAdapterManager::getScalarOutputAdapter( CspTypePtr &type, const Dictionary &properties ) +OutputAdapter * ParquetOutputAdapterManager::getScalarOutputAdapter( CspTypePtr & type, const Dictionary & properties ) { auto columnName = properties.get( "column_name" ); - return m_parquetWriter -> getScalarOutputAdapter( type, columnName ); + return m_parquetWriter->getScalarOutputAdapter( type, columnName ); } -OutputAdapter *ParquetOutputAdapterManager::getStructOutputAdapter( CspTypePtr &type, const Dictionary &properties ) +OutputAdapter * ParquetOutputAdapterManager::getStructOutputAdapter( CspTypePtr & type, const Dictionary & properties ) { auto fieldMap = properties.get( "field_map" ); - return m_parquetWriter -> getStructOutputAdapter( type, fieldMap ); + return m_parquetWriter->getStructOutputAdapter( type, fieldMap ); } - -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.cpp b/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.cpp index 968d68f2..0ab071a3 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.cpp +++ b/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include namespace csp::adapters::parquet @@ -7,7 +7,7 @@ namespace csp::adapters::parquet void ParquetOutputFilenameAdapter::executeImpl() { - m_parquetOutputAdapterManager.changeFileName( input()->lastValueTyped()); + m_parquetOutputAdapterManager.changeFileName( input()->lastValueTyped() ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetReader.cpp b/cpp/csp/adapters/parquet/ParquetReader.cpp index 81cfd40e..3b9988e2 100644 --- a/cpp/csp/adapters/parquet/ParquetReader.cpp +++ b/cpp/csp/adapters/parquet/ParquetReader.cpp @@ -1,15 +1,15 @@ -#include +#include +#include +#include #include -#include #include +#include +#include #include -#include #include +#include #include #include -#include -#include -#include namespace { @@ -21,50 +21,50 @@ struct FieldInfo std::size_t getFieldWidth( std::shared_ptr<::arrow::DataType> fieldType ) { - if( fieldType -> id() != ::arrow::Type::STRUCT ) + if( fieldType->id() != ::arrow::Type::STRUCT ) { return 1; } std::size_t res = 0; - for( auto &childField: fieldType -> fields() ) + for( auto & childField : fieldType->fields() ) { - res += getFieldWidth( childField -> type() ); + res += getFieldWidth( childField->type() ); } return res; } -inline std::vector getFieldsInfo( const std::vector> &fields ) +inline std::vector getFieldsInfo( const std::vector> & fields ) { - std::size_t curStartIndex = 0; + std::size_t curStartIndex = 0; std::vector res; res.reserve( fields.size() ); - for( auto &field:fields ) + for( auto & field : fields ) { - auto width = getFieldWidth( field -> type() ); + auto width = getFieldWidth( field->type() ); res.push_back( { curStartIndex, width } ); curStartIndex += width; } return res; } -} +} // namespace namespace csp::adapters::parquet { -ParquetStructAdapter &ParquetReader::getStructAdapter( const StructAdapterInfo &structAdapterInfo ) +ParquetStructAdapter & ParquetReader::getStructAdapter( const StructAdapterInfo & structAdapterInfo ) { auto it = m_structInfoToAdapterIndex.find( structAdapterInfo ); if( it == m_structInfoToAdapterIndex.end() ) { - m_structInfoToAdapterIndex[ structAdapterInfo ] = m_structAdapters.size(); + m_structInfoToAdapterIndex[structAdapterInfo] = m_structAdapters.size(); m_structAdapters.push_back( std::make_unique( *this, structAdapterInfo ) ); - return *m_structAdapters[ m_structAdapters.size() - 1 ]; + return *m_structAdapters[m_structAdapters.size() - 1]; } else { - return *m_structAdapters[ it -> second ]; + return *m_structAdapters[it->second]; } } @@ -76,20 +76,20 @@ const utils::Symbol * ParquetReader::getCurSymbol() { case CspType::Type::STRING: { - auto &curSymbol = getSymbolColumnAdapter() -> getCurValue(); + auto & curSymbol = getSymbolColumnAdapter()->getCurValue(); CSP_TRUE_OR_THROW_RUNTIME( curSymbol.has_value(), "Parquet file row contains row with no value for symbol column " - << getSymbolColumnName().value() ); + << getSymbolColumnName().value() ); m_curSymbol = curSymbol.value(); break; } case CspType::Type::INT64: { - auto &curSymbol = getSymbolColumnAdapter() -> getCurValue(); + auto & curSymbol = getSymbolColumnAdapter()->getCurValue(); CSP_TRUE_OR_THROW_RUNTIME( curSymbol.has_value(), "Parquet file row contains row with no value for symbol column " - << getSymbolColumnName().value() ); + << getSymbolColumnName().value() ); m_curSymbol = curSymbol.value(); break; } @@ -104,25 +104,25 @@ const utils::Symbol * ParquetReader::getCurSymbol() return nullptr; } - void ParquetReader::setSymbolColumnAdapter( ColumnAdapterReference adapter ) { m_symbolColumn = adapter; try { - m_symbolColumn -> ensureType( CspType::STRING() ); + m_symbolColumn->ensureType( CspType::STRING() ); m_symbolType = CspType::Type::STRING; } catch( const TypeError & ) { try { - m_symbolColumn -> ensureType( CspType::INT64() ); + m_symbolColumn->ensureType( CspType::INT64() ); m_symbolType = CspType::Type::INT64; } catch( const TypeError & ) { - CSP_THROW( TypeError, "Invalid symbol column type. Only string and int64 symbols are currently supported" ); + CSP_THROW( TypeError, + "Invalid symbol column type. Only string and int64 symbols are currently supported" ); } } } @@ -132,26 +132,30 @@ void ParquetReader::validateSymbolType( const utils::Symbol & symbol ) switch( m_symbolType ) { case CspType::Type::STRING: - CSP_TRUE_OR_THROW( std::holds_alternative( symbol ), TypeError, "Provided symbol type does not match symbol column type (string)" ); + CSP_TRUE_OR_THROW( std::holds_alternative( symbol ), TypeError, + "Provided symbol type does not match symbol column type (string)" ); break; case CspType::Type::INT64: - CSP_TRUE_OR_THROW( std::holds_alternative( symbol ), TypeError, "Provided symbol type does not match symbol column type (int64)" ); + CSP_TRUE_OR_THROW( std::holds_alternative( symbol ), TypeError, + "Provided symbol type does not match symbol column type (int64)" ); break; default: CSP_THROW( RuntimeException, "Unexpected symbol type: " << m_symbolType ); } } -SingleTableParquetReader::SingleTableParquetReader( std::vector columns, bool arrowIPC, bool allowMissingColumns, +SingleTableParquetReader::SingleTableParquetReader( std::vector columns, bool arrowIPC, + bool allowMissingColumns, std::optional symbolColumnName ) - : ParquetReader( symbolColumnName, arrowIPC, allowMissingColumns ), m_columns( columns ) + : ParquetReader( symbolColumnName, arrowIPC, allowMissingColumns ) + , m_columns( columns ) { } void SingleTableParquetReader::init() { - if(!openNextFile()) + if( !openNextFile() ) { return; } @@ -166,31 +170,31 @@ void SingleTableParquetReader::setColumnAdaptersFromCurrentTable() m_neededColumnIndices.reserve( m_columns.size() ); m_columnAdapters.reserve( m_columns.size() ); - auto &fields = m_schema -> fields(); + auto & fields = m_schema->fields(); auto fieldsInfo = getFieldsInfo( fields ); - for( const auto &columnName:m_columns ) + for( const auto & columnName : m_columns ) { - auto index = m_schema -> GetFieldIndex( columnName ); - auto existingRecordIt = m_columnNameToAdapterIndex.find(columnName); - if(existingRecordIt != m_columnNameToAdapterIndex.end()) + auto index = m_schema->GetFieldIndex( columnName ); + auto existingRecordIt = m_columnNameToAdapterIndex.find( columnName ); + if( existingRecordIt != m_columnNameToAdapterIndex.end() ) { - CSP_TRUE_OR_THROW_RUNTIME( existingRecordIt -> second == m_columnAdapters.size(), - "Unexpected index change of column " << columnName << - " was " << existingRecordIt -> second << " became " + CSP_TRUE_OR_THROW_RUNTIME( existingRecordIt->second == m_columnAdapters.size(), + "Unexpected index change of column " << columnName << " was " + << existingRecordIt->second << " became " << m_columnAdapters.size() ); } else { - m_columnNameToAdapterIndex[ columnName ] = m_columnAdapters.size(); + m_columnNameToAdapterIndex[columnName] = m_columnAdapters.size(); } std::unique_ptr columnAdapter; if( index >= 0 ) { - auto &field = fields[ index ]; - columnAdapter = createColumnAdapter( *this, *field, getCurFileOrTableName(), &getStructColumnMeta() ); - auto &fieldInfo = fieldsInfo[ index ]; + auto & field = fields[index]; + columnAdapter = createColumnAdapter( *this, *field, getCurFileOrTableName(), &getStructColumnMeta() ); + auto & fieldInfo = fieldsInfo[index]; for( std::size_t i = 0; i < fieldInfo.m_width; ++i ) { @@ -199,7 +203,8 @@ void SingleTableParquetReader::setColumnAdaptersFromCurrentTable() } else { - CSP_TRUE_OR_THROW_RUNTIME( isAllowMissingColumns(), "Missing column " << columnName << " in file " << getCurFileOrTableName() ); + CSP_TRUE_OR_THROW_RUNTIME( isAllowMissingColumns(), + "Missing column " << columnName << " in file " << getCurFileOrTableName() ); columnAdapter = createMissingColumnAdapter( *this, columnName ); } m_columnAdapters.push_back( std::move( columnAdapter ) ); @@ -207,20 +212,13 @@ void SingleTableParquetReader::setColumnAdaptersFromCurrentTable() if( getSymbolColumnName().has_value() ) { - setSymbolColumnAdapter( ( *this )[ getSymbolColumnName().value() ] ); + setSymbolColumnAdapter( ( *this )[getSymbolColumnName().value()] ); } } +bool SingleTableParquetReader::start() { return readNextRowGroup() && readNextRow(); } -bool SingleTableParquetReader::start() -{ - return readNextRowGroup() && readNextRow(); -} - -bool SingleTableParquetReader::skipRow() -{ - return readNextRow(); -} +bool SingleTableParquetReader::skipRow() { return readNextRow(); } bool SingleTableParquetReader::readNextRow() { @@ -229,38 +227,33 @@ bool SingleTableParquetReader::readNextRow() return false; } - if( unlikely( m_curTableNextRow >= m_curTable -> num_rows() ) ) + if( unlikely( m_curTableNextRow >= m_curTable->num_rows() ) ) { if( !readNextRowGroup() ) { return false; } } - for( auto &colAdapter:m_columnAdapters ) + for( auto & colAdapter : m_columnAdapters ) { - colAdapter -> readCurValue(); + colAdapter->readCurValue(); } ++m_curTableNextRow; return true; } +void SingleTableParquetReader::dispatchRow( bool doReadNextRow ) { dispatchRow( doReadNextRow, getCurSymbol() ); } -void SingleTableParquetReader::dispatchRow( bool doReadNextRow ) +void SingleTableParquetReader::dispatchRow( bool doReadNextRow, const utils::Symbol * symbol ) { - - dispatchRow( doReadNextRow, getCurSymbol() ); -} - -void SingleTableParquetReader::dispatchRow( bool doReadNextRow, const utils::Symbol *symbol ) -{ - for( auto &adapter:m_columnAdapters ) + for( auto & adapter : m_columnAdapters ) { - adapter -> dispatchValue( symbol ); + adapter->dispatchValue( symbol ); } - for( auto &adapter:getStructAdapters() ) + for( auto & adapter : getStructAdapters() ) { - adapter -> dispatchValue( symbol ); + adapter->dispatchValue( symbol ); } if( doReadNextRow ) { @@ -268,10 +261,7 @@ void SingleTableParquetReader::dispatchRow( bool doReadNextRow, const utils::Sym } } -bool SingleTableParquetReader::hasData() const -{ - return m_curTable != nullptr; -} +bool SingleTableParquetReader::hasData() const { return m_curTable != nullptr; } void SingleTableParquetReader::clear() { @@ -282,23 +272,26 @@ void SingleTableParquetReader::clear() m_curTableNextRow = -1; } -SingleFileParquetReader::SingleFileParquetReader( GeneratorPtr generatorPtr, std::vector columns, bool arrowIPC, - bool allowMissingColumns, bool allowMissingFiles, std::optional symbolColumnName ) - : SingleTableParquetReader( columns, arrowIPC, allowMissingColumns, symbolColumnName ), m_generatorPtr( generatorPtr ), m_allowMissingFiles(allowMissingFiles) +SingleFileParquetReader::SingleFileParquetReader( GeneratorPtr generatorPtr, std::vector columns, + bool arrowIPC, bool allowMissingColumns, bool allowMissingFiles, + std::optional symbolColumnName ) + : SingleTableParquetReader( columns, arrowIPC, allowMissingColumns, symbolColumnName ) + , m_generatorPtr( generatorPtr ) + , m_allowMissingFiles( allowMissingFiles ) { init(); } bool SingleFileParquetReader::openNextFile() { - std::string fileName; - FileReaderWrapperPtr fileReader; + std::string fileName; + FileReaderWrapperPtr fileReader; std::shared_ptr<::arrow::Schema> fileSchema; - while(true) + while( true ) { - if( m_generatorPtr -> next( fileName ) ) + if( m_generatorPtr->next( fileName ) ) { - if(m_allowMissingFiles && !csp::utils::fileExists(fileName)) + if( m_allowMissingFiles && !csp::utils::fileExists( fileName ) ) { continue; } @@ -310,8 +303,8 @@ bool SingleFileParquetReader::openNextFile() { fileReader = std::make_unique(); } - fileReader -> open( fileName ); - fileReader -> getSchema( fileSchema ); + fileReader->open( fileName ); + fileReader->getSchema( fileSchema ); break; } else @@ -321,12 +314,12 @@ bool SingleFileParquetReader::openNextFile() } } - auto is_new_schema = m_schema && !m_schema -> Equals( *fileSchema ); - m_fileName = fileName; - m_fileReader = std::move( fileReader ); - m_schema = fileSchema; + auto is_new_schema = m_schema && !m_schema->Equals( *fileSchema ); + m_fileName = fileName; + m_fileReader = std::move( fileReader ); + m_schema = fileSchema; - if(is_new_schema) + if( is_new_schema ) { setColumnAdaptersFromCurrentTable(); resubscribeAll(); @@ -345,7 +338,7 @@ bool SingleFileParquetReader::readNextRowGroup() while( !endOfData ) { // Get to first non empty table - while( m_fileReader -> readNextRowGroup( m_neededColumnIndices, m_curTable ) && m_curTable -> num_rows() == 0 ) + while( m_fileReader->readNextRowGroup( m_neededColumnIndices, m_curTable ) && m_curTable->num_rows() == 0 ) { } if( m_curTable != nullptr ) @@ -360,14 +353,14 @@ bool SingleFileParquetReader::readNextRowGroup() return false; } m_curTableNextRow = 0; - auto columns = m_curTable -> columns(); + auto columns = m_curTable->columns(); std::size_t columnIndex = 0; for( std::size_t i = 0; i < m_columnAdapters.size(); ++i ) { - if(!m_columnAdapters[i]->isMissingColumn()) + if( !m_columnAdapters[i]->isMissingColumn() ) { - m_columnAdapters[ i ] -> handleNewBatch( columns[ columnIndex++ ] ); + m_columnAdapters[i]->handleNewBatch( columns[columnIndex++] ); } } return true; @@ -382,8 +375,10 @@ void SingleFileParquetReader::clear() } InMemoryTableParquetReader::InMemoryTableParquetReader( GeneratorPtr generatorPtr, std::vector columns, - bool allowMissingColumns, std::optional symbolColumnName ) - : SingleTableParquetReader( columns, true, allowMissingColumns, symbolColumnName ), m_generatorPtr( generatorPtr ) + bool allowMissingColumns, + std::optional symbolColumnName ) + : SingleTableParquetReader( columns, true, allowMissingColumns, symbolColumnName ) + , m_generatorPtr( generatorPtr ) { init(); } @@ -391,33 +386,34 @@ InMemoryTableParquetReader::InMemoryTableParquetReader( GeneratorPtr generatorPt bool InMemoryTableParquetReader::openNextFile() { std::shared_ptr<::arrow::Schema> schema; - std::shared_ptr<::arrow::Table> table; + std::shared_ptr<::arrow::Table> table; - if( !m_generatorPtr -> next( table ) ) + if( !m_generatorPtr->next( table ) ) { clear(); return false; } - CSP_TRUE_OR_THROW_RUNTIME( table -> num_columns() > 0, "Provided in memory arrow table with 0 columns" ); - schema = table -> schema(); + CSP_TRUE_OR_THROW_RUNTIME( table->num_columns() > 0, "Provided in memory arrow table with 0 columns" ); + schema = table->schema(); - int refNumChunks = table -> column( 0 ) -> num_chunks(); + int refNumChunks = table->column( 0 )->num_chunks(); - for( int i = 0; i < table -> num_columns(); ++i ) + for( int i = 0; i < table->num_columns(); ++i ) { - CSP_TRUE_OR_THROW_RUNTIME( table -> column( i ) -> num_chunks() == refNumChunks, + CSP_TRUE_OR_THROW_RUNTIME( table->column( i )->num_chunks() == refNumChunks, "Found in memory table with non aligned chunks, number of chunks in one column is " - << refNumChunks << " vs " << table -> column( i ) -> num_chunks() << " in another table" ); + << refNumChunks << " vs " << table->column( i )->num_chunks() + << " in another table" ); } - auto is_new_schema = m_schema && !m_schema -> Equals( *schema ); + auto is_new_schema = m_schema && !m_schema->Equals( *schema ); m_schema = schema; m_fullTable = table; m_nextChunkIndex = 0; m_curTable = nullptr; - if(is_new_schema) + if( is_new_schema ) { setColumnAdaptersFromCurrentTable(); resubscribeAll(); @@ -436,33 +432,33 @@ bool InMemoryTableParquetReader::readNextRowGroup() while( !endOfData ) { - if(m_nextChunkIndex >= m_fullTable->column(0)->num_chunks()) + if( m_nextChunkIndex >= m_fullTable->column( 0 )->num_chunks() ) { endOfData = !openNextFile(); continue; } std::vector> curTableChunks; - auto refChunkLength = m_fullTable->column(0)->chunk(m_nextChunkIndex)->length(); + auto refChunkLength = m_fullTable->column( 0 )->chunk( m_nextChunkIndex )->length(); std::vector> neededFields; - for(auto colIndex : m_neededColumnIndices) + for( auto colIndex : m_neededColumnIndices ) { - neededFields.push_back(m_fullTable->field(colIndex)); - auto &&curArrayChunk = m_fullTable -> column( colIndex ) -> chunk( m_nextChunkIndex ); - CSP_TRUE_OR_THROW_RUNTIME( curArrayChunk -> length() == refChunkLength, + neededFields.push_back( m_fullTable->field( colIndex ) ); + auto && curArrayChunk = m_fullTable->column( colIndex )->chunk( m_nextChunkIndex ); + CSP_TRUE_OR_THROW_RUNTIME( curArrayChunk->length() == refChunkLength, "Found in memory table with non aligned chunks, for chunk " - << m_nextChunkIndex << "found arrays of lenght " << refChunkLength << " and " - << curArrayChunk -> length() ); - arrow::Result> newChunkedArrayResult = arrow::ChunkedArray::Make( - std::vector>{curArrayChunk}); - STATUS_OK_OR_THROW_RUNTIME(newChunkedArrayResult.status(), "Failed to creae a new chunked array"); - curTableChunks.push_back(newChunkedArrayResult.ValueUnsafe()); + << m_nextChunkIndex << "found arrays of lenght " << refChunkLength << " and " + << curArrayChunk->length() ); + arrow::Result> newChunkedArrayResult + = arrow::ChunkedArray::Make( std::vector>{ curArrayChunk } ); + STATUS_OK_OR_THROW_RUNTIME( newChunkedArrayResult.status(), "Failed to creae a new chunked array" ); + curTableChunks.push_back( newChunkedArrayResult.ValueUnsafe() ); } m_nextChunkIndex += 1; - if(refChunkLength > 0) + if( refChunkLength > 0 ) { - m_curTable = arrow::Table::Make(arrow::schema(neededFields), curTableChunks); + m_curTable = arrow::Table::Make( arrow::schema( neededFields ), curTableChunks ); break; } } @@ -472,14 +468,14 @@ bool InMemoryTableParquetReader::readNextRowGroup() return false; } m_curTableNextRow = 0; - auto columns = m_curTable -> columns(); + auto columns = m_curTable->columns(); std::size_t columnIndex = 0; for( std::size_t i = 0; i < m_columnAdapters.size(); ++i ) { - if(!m_columnAdapters[i]->isMissingColumn()) + if( !m_columnAdapters[i]->isMissingColumn() ) { - m_columnAdapters[ i ] -> handleNewBatch( columns[ columnIndex++ ] ); + m_columnAdapters[i]->handleNewBatch( columns[columnIndex++] ); } } return true; @@ -501,25 +497,22 @@ void InMemoryTableParquetReader::clear() } MultipleFileParquetReader::MultipleFileParquetReader( FileNameGeneratorReplicator::Ptr generatorReplicatorPtr, - std::vector columns, - bool arrowIPC, + std::vector columns, bool arrowIPC, bool allowMissingColumns, std::optional symbolColumnName ) - : ParquetReader( symbolColumnName, arrowIPC, allowMissingColumns ), m_generatorReplicatorPtr( generatorReplicatorPtr ) + : ParquetReader( symbolColumnName, arrowIPC, allowMissingColumns ) + , m_generatorReplicatorPtr( generatorReplicatorPtr ) { - for( auto &&column : columns ) - { - m_columnReaders.push_back( - std::make_unique( - m_generatorReplicatorPtr -> getGeneratorReplica( std::string( "/" ) + '/' + column + ".parquet" ), - std::vector{ column }, - arrowIPC, - allowMissingColumns) ); + for( auto && column : columns ) + { + m_columnReaders.push_back( std::make_unique( + m_generatorReplicatorPtr->getGeneratorReplica( std::string( "/" ) + '/' + column + ".parquet" ), + std::vector{ column }, arrowIPC, allowMissingColumns ) ); m_columnReaderByName[column] = m_columnReaders.back().get(); } if( symbolColumnName.has_value() ) { - setSymbolColumnAdapter( ( *this )[ symbolColumnName.value() ] ); + setSymbolColumnAdapter( ( *this )[symbolColumnName.value()] ); } } @@ -527,90 +520,80 @@ bool MultipleFileParquetReader::start() { uint32_t successCount = 0; - for( auto &&columnReader : m_columnReaders ) + for( auto && columnReader : m_columnReaders ) { - if( columnReader -> start() ) + if( columnReader->start() ) { ++successCount; } } CSP_TRUE_OR_THROW_RUNTIME( successCount == 0 || successCount == m_columnReaders.size(), - "Expected all or none of the column readers to start, actual:" << successCount << '/' - << m_columnReaders.size() ); + "Expected all or none of the column readers to start, actual:" + << successCount << '/' << m_columnReaders.size() ); return successCount != 0; } - -ColumnAdapterReference MultipleFileParquetReader::operator[]( const std::string &name ) +ColumnAdapterReference MultipleFileParquetReader::operator[]( const std::string & name ) { - auto it = m_columnReaderByName.find(name); - CSP_TRUE_OR_THROW_RUNTIME( it != m_columnReaderByName.end(), - "No column " << name << " found in parquet file" ); - return (*it -> second)[name]; + auto it = m_columnReaderByName.find( name ); + CSP_TRUE_OR_THROW_RUNTIME( it != m_columnReaderByName.end(), "No column " << name << " found in parquet file" ); + return ( *it->second )[name]; } -ParquetColumnAdapter* MultipleFileParquetReader::getCurrentColumnAdapterByIndex( std::size_t index ) +ParquetColumnAdapter * MultipleFileParquetReader::getCurrentColumnAdapterByIndex( std::size_t index ) { CSP_NOT_IMPLEMENTED; } - bool MultipleFileParquetReader::skipRow() { unsigned successfulSkips = 0; - for( auto &&columnReader : m_columnReaders ) + for( auto && columnReader : m_columnReaders ) { - if(columnReader -> skipRow()) + if( columnReader->skipRow() ) { ++successfulSkips; } } - if(unlikely(successfulSkips == 0)) + if( unlikely( successfulSkips == 0 ) ) { return false; } - if(unlikely(successfulSkips!=m_columnReaders.size())) + if( unlikely( successfulSkips != m_columnReaders.size() ) ) { - CSP_THROW(RuntimeException, "Input files are not alligned - some columns have more data than the others"); + CSP_THROW( RuntimeException, "Input files are not alligned - some columns have more data than the others" ); } return true; } void MultipleFileParquetReader::dispatchRow( bool doReadNextRow ) { - // By default dispatchRow of columnReaders will dispatch row and read the next row (clear out the currently read row). - // If we have structAdapters, we can't move on to the next row before we populate the struct adapter fields. So if any - // struct adapters exist, we need to postpone reading of next row for child readers and explicitly call them after dispatching the - // struct values. - auto *symbol = getCurSymbol(); + // By default dispatchRow of columnReaders will dispatch row and read the next row (clear out the currently read + // row). If we have structAdapters, we can't move on to the next row before we populate the struct adapter fields. + // So if any struct adapters exist, we need to postpone reading of next row for child readers and explicitly call + // them after dispatching the struct values. + auto * symbol = getCurSymbol(); bool childDoReadNextRow = getStructAdapters().empty() && doReadNextRow && symbol == nullptr; - for( auto &&columnReader : m_columnReaders ) + for( auto && columnReader : m_columnReaders ) { - columnReader -> dispatchRow( childDoReadNextRow, symbol ); + columnReader->dispatchRow( childDoReadNextRow, symbol ); } - for( auto &adapter:getStructAdapters() ) + for( auto & adapter : getStructAdapters() ) { - adapter -> dispatchValue( symbol ); + adapter->dispatchValue( symbol ); } if( !childDoReadNextRow && doReadNextRow ) { - for( auto &&columnReader : m_columnReaders ) + for( auto && columnReader : m_columnReaders ) { - columnReader -> readNextRow(); + columnReader->readNextRow(); } } } -bool MultipleFileParquetReader::hasData() const -{ - return !m_columnReaders.empty() && m_columnReaders[ 0 ] -> hasData(); -} - -void MultipleFileParquetReader::clear() -{ - m_columnReaders.clear(); -} +bool MultipleFileParquetReader::hasData() const { return !m_columnReaders.empty() && m_columnReaders[0]->hasData(); } +void MultipleFileParquetReader::clear() { m_columnReaders.clear(); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp b/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp index 3fe763a6..0e0672b6 100644 --- a/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp +++ b/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp @@ -1,157 +1,158 @@ -#include #include +#include #include +#include #include #include -#include namespace csp::adapters::parquet { -template< typename ArrowArrayType > -static inline std::unique_ptr createDateColumnAdapter( - ParquetReader &reader, - const std::string& columnName, - const std::shared_ptr &dateType ) +template +static inline std::unique_ptr +createDateColumnAdapter( ParquetReader & reader, const std::string & columnName, + const std::shared_ptr & dateType ) { - switch( dateType -> unit() ) + switch( dateType->unit() ) { case arrow::DateUnit::MILLI: return std::unique_ptr( - new DateColumnAdapter<1000000, ArrowArrayType>( reader, columnName ) ); + new DateColumnAdapter<1000000, ArrowArrayType>( reader, columnName ) ); case arrow::DateUnit::DAY: return std::unique_ptr( - new DateColumnAdapter<1000000000LL * 3600 * 24, ArrowArrayType>( reader, columnName ) ); + new DateColumnAdapter<1000000000LL * 3600 * 24, ArrowArrayType>( reader, columnName ) ); } - CSP_THROW( csp::TypeError, "Unexpected day unit: " << ( int ) dateType -> unit() << " for column " << columnName ); + CSP_THROW( csp::TypeError, "Unexpected day unit: " << (int)dateType->unit() << " for column " << columnName ); } -template< typename ArrowArrayType > -static inline std::unique_ptr createTimeColumnAdapter( - ParquetReader &reader, - const std::string& columnName, - const std::shared_ptr &timeType ) +template +static inline std::unique_ptr +createTimeColumnAdapter( ParquetReader & reader, const std::string & columnName, + const std::shared_ptr & timeType ) { ParquetColumnAdapter * adapter; - switch( timeType -> unit() ) - { - case arrow::TimeUnit::SECOND: adapter = new TimeColumnAdapter<1000000000,ArrowArrayType>( reader, columnName ); break; - case arrow::TimeUnit::MILLI: adapter = new TimeColumnAdapter<1000000,ArrowArrayType>( reader, columnName ); break; - case arrow::TimeUnit::MICRO: adapter = new TimeColumnAdapter<1000,ArrowArrayType>( reader, columnName ); break; - case arrow::TimeUnit::NANO: adapter = new TimeColumnAdapter<1,ArrowArrayType>( reader, columnName ); break; + switch( timeType->unit() ) + { + case arrow::TimeUnit::SECOND: + adapter = new TimeColumnAdapter<1000000000, ArrowArrayType>( reader, columnName ); + break; + case arrow::TimeUnit::MILLI: + adapter = new TimeColumnAdapter<1000000, ArrowArrayType>( reader, columnName ); + break; + case arrow::TimeUnit::MICRO: + adapter = new TimeColumnAdapter<1000, ArrowArrayType>( reader, columnName ); + break; + case arrow::TimeUnit::NANO: + adapter = new TimeColumnAdapter<1, ArrowArrayType>( reader, columnName ); + break; } return std::unique_ptr( adapter ); } -std::unique_ptr createColumnAdapter( - ParquetReader &parquetReader, - const ::arrow::Field &field, - const std::string &fileName, - const std::map>* structMetaByColumnName) +std::unique_ptr +createColumnAdapter( ParquetReader & parquetReader, const ::arrow::Field & field, const std::string & fileName, + const std::map> * structMetaByColumnName ) { - auto typeId = field.type() -> id(); + auto typeId = field.type()->id(); switch( typeId ) { case arrow::Type::BOOL: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::UINT8: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::INT8: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::UINT16: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::INT16: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::UINT32: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::INT32: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::UINT64: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::INT64: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::HALF_FLOAT: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::FLOAT: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::DOUBLE: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::STRING: return std::unique_ptr( new StringColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::BINARY: return std::unique_ptr( new BytesColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::DATE32: - return createDateColumnAdapter( parquetReader, - field.name(), - std::static_pointer_cast( - field.type() ) ); + return createDateColumnAdapter( + parquetReader, field.name(), std::static_pointer_cast( field.type() ) ); case arrow::Type::DATE64: - return createDateColumnAdapter( parquetReader, - field.name(), - std::static_pointer_cast( - field.type() ) ); + return createDateColumnAdapter( + parquetReader, field.name(), std::static_pointer_cast( field.type() ) ); case arrow::Type::TIME32: - return createTimeColumnAdapter( parquetReader, field.name(), - std::static_pointer_cast( field.type() ) ); + return createTimeColumnAdapter( + parquetReader, field.name(), std::static_pointer_cast( field.type() ) ); case arrow::Type::TIME64: - return createTimeColumnAdapter( parquetReader, field.name(), - std::static_pointer_cast( field.type() ) ); + return createTimeColumnAdapter( + parquetReader, field.name(), std::static_pointer_cast( field.type() ) ); case arrow::Type::TIMESTAMP: { auto timestampType = std::static_pointer_cast( field.type() ); -// CSP_TRUE_OR_THROW_RUNTIME( -// timestampType -> timezone().empty() || timestampType -> timezone() == "UTC" || timestampType -> timezone() == "utc", -// "Unexpected parquet column timezone: " << timestampType -> timezone() ); - switch( timestampType -> unit() ) + // CSP_TRUE_OR_THROW_RUNTIME( + // timestampType -> timezone().empty() || timestampType -> timezone() == "UTC" || + // timestampType -> timezone() == "utc", "Unexpected parquet column timezone: " << + // timestampType -> timezone() ); + switch( timestampType->unit() ) { case arrow::TimeUnit::SECOND: return std::unique_ptr( - new DatetimeColumnAdapter<1000000000>( parquetReader, field.name() ) ); + new DatetimeColumnAdapter<1000000000>( parquetReader, field.name() ) ); case arrow::TimeUnit::MILLI: return std::unique_ptr( - new DatetimeColumnAdapter<1000000>( parquetReader, field.name() ) ); + new DatetimeColumnAdapter<1000000>( parquetReader, field.name() ) ); case arrow::TimeUnit::MICRO: return std::unique_ptr( - new DatetimeColumnAdapter<1000>( parquetReader, field.name() ) ); + new DatetimeColumnAdapter<1000>( parquetReader, field.name() ) ); case arrow::TimeUnit::NANO: return std::unique_ptr( - new DatetimeColumnAdapter<1>( parquetReader, field.name() ) ); + new DatetimeColumnAdapter<1>( parquetReader, field.name() ) ); } } case arrow::Type::DURATION: { auto durationType = std::static_pointer_cast( field.type() ); - switch( durationType -> unit() ) + switch( durationType->unit() ) { case arrow::TimeUnit::SECOND: return std::unique_ptr( - new DurationColumnAdapter<1000000000>( parquetReader, field.name() ) ); + new DurationColumnAdapter<1000000000>( parquetReader, field.name() ) ); case arrow::TimeUnit::MILLI: return std::unique_ptr( - new DurationColumnAdapter<1000000>( parquetReader, field.name() ) ); + new DurationColumnAdapter<1000000>( parquetReader, field.name() ) ); case arrow::TimeUnit::MICRO: return std::unique_ptr( - new DurationColumnAdapter<1000>( parquetReader, field.name() ) ); + new DurationColumnAdapter<1000>( parquetReader, field.name() ) ); case arrow::TimeUnit::NANO: return std::unique_ptr( - new DurationColumnAdapter<1>( parquetReader, field.name() ) ); + new DurationColumnAdapter<1>( parquetReader, field.name() ) ); } } case arrow::Type::FIXED_SIZE_BINARY: @@ -162,24 +163,24 @@ std::unique_ptr createColumnAdapter( { auto dictionaryType = std::static_pointer_cast( field.type() ); - if( dictionaryType -> value_type() -> id() != arrow::Type::STRING ) + if( dictionaryType->value_type()->id() != arrow::Type::STRING ) { CSP_THROW( ParquetColumnTypeError, - "Unsupported dictionary column type " << field.type() -> name() << " in file " << fileName + "Unsupported dictionary column type " << field.type()->name() << " in file " << fileName << " only string values supported" ); } return std::make_unique( parquetReader, field.name() ); } case arrow::Type::STRUCT: { - auto res = std::make_unique( parquetReader, std::static_pointer_cast<::arrow::StructType>( field.type() ), - field.name() ); - if(structMetaByColumnName!= nullptr) + auto res = std::make_unique( + parquetReader, std::static_pointer_cast<::arrow::StructType>( field.type() ), field.name() ); + if( structMetaByColumnName != nullptr ) { - auto metaIt = structMetaByColumnName->find(field.name() ); - if(metaIt != structMetaByColumnName->end()) + auto metaIt = structMetaByColumnName->find( field.name() ); + if( metaIt != structMetaByColumnName->end() ) { - res->setStructMeta(metaIt->second); + res->setStructMeta( metaIt->second ); } } return res; @@ -187,8 +188,8 @@ std::unique_ptr createColumnAdapter( case arrow::Type::LIST: { auto listType = std::static_pointer_cast( field.type() ); - auto valueType = listType -> value_type(); - switch( valueType -> id() ) + auto valueType = listType->value_type(); + switch( valueType->id() ) { case arrow::Type::INT64: return std::make_unique>( parquetReader, field.name() ); @@ -197,58 +198,65 @@ std::unique_ptr createColumnAdapter( case arrow::Type::BOOL: return std::make_unique>( parquetReader, field.name() ); case arrow::Type::STRING: - return std::make_unique>( parquetReader, field.name() ); + return std::make_unique>( parquetReader, + field.name() ); default: - CSP_THROW( TypeError, "Trying to create arrow list array reader for unsupported element type " << valueType -> name() << " for column " << field.name() ); + CSP_THROW( TypeError, + "Trying to create arrow list array reader for unsupported element type " + << valueType->name() << " for column " << field.name() ); } } case arrow::Type::LARGE_STRING: default: CSP_THROW( ParquetColumnTypeError, - "Unsupported column type " << field.type() -> name() << " for column " << field.name() << " in file " << fileName ); + "Unsupported column type " << field.type()->name() << " for column " << field.name() + << " in file " << fileName ); } } -std::unique_ptr createMissingColumnAdapter(ParquetReader &parquetReader,const std::string& columnName) +std::unique_ptr createMissingColumnAdapter( ParquetReader & parquetReader, + const std::string & columnName ) { return std::make_unique( parquetReader, columnName ); } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > -void BaseTypedColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, - std::optional symbol ) +template +void BaseTypedColumnAdapter::addSubscriber( + ManagedSimInputAdapter * inputAdapter, std::optional symbol ) { try { - auto callback = CompatibleTypeSwitch::invoke( inputAdapter -> type(), [ inputAdapter ]( auto tag ) - { - return std::function( [ inputAdapter ]( const ValueType *val ) - { - if( val ) - { - inputAdapter -> pushTick( *val ); - } - else - { - inputAdapter -> pushNullTick(); - } - } ); - - } ); + auto callback = CompatibleTypeSwitch::invoke( + inputAdapter->type(), + [inputAdapter]( auto tag ) + { + return std::function( + [inputAdapter]( const ValueType * val ) + { + if( val ) + { + inputAdapter->pushTick( *val ); + } + else + { + inputAdapter->pushNullTick(); + } + } ); + } ); m_dispatcher.addSubscriber( callback, symbol ); } - catch( UnsupportedSwitchType &e ) + catch( UnsupportedSwitchType & e ) { - CSP_THROW( TypeError, "Unexpected column type for column " << getColumnName() << " , expected " - << inputAdapter -> type() -> type().asCString() << - " got " << ArrowArrayType::TypeClass::type_name() ); + CSP_THROW( TypeError, + "Unexpected column type for column " << getColumnName() << " , expected " + << inputAdapter->type()->type().asCString() << " got " + << ArrowArrayType::TypeClass::type_name() ); } } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > -void BaseTypedColumnAdapter::dispatchValue( const utils::Symbol *symbol ) +template +void BaseTypedColumnAdapter::dispatchValue( const utils::Symbol * symbol ) { if( m_curValue.has_value() ) { @@ -260,217 +268,223 @@ void BaseTypedColumnAdapter::dispat } } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > +template void BaseTypedColumnAdapter::ensureType( CspType::Ptr cspType ) { try { - CompatibleTypeSwitch::invoke( cspType.get(), [ cspType, this ]( auto tag ) - { - // No need to check native types here, they won't be assignable so it will raise exception. - if( cspType -> type() > CspType::Type::MAX_NATIVE_TYPE ) + CompatibleTypeSwitch::invoke( + cspType.get(), + [cspType, this]( auto tag ) { - CSP_TRUE_OR_THROW( ( std::is_same::value ), TypeError, - "Unexpected column type for column " << getColumnName() << " , expected " << cspType -> type().asCString() << - " got " << ArrowArrayType::TypeClass::type_name() ); - } - } ); + // No need to check native types here, they won't be assignable so it will raise exception. + if( cspType->type() > CspType::Type::MAX_NATIVE_TYPE ) + { + CSP_TRUE_OR_THROW( ( std::is_same::value ), TypeError, + "Unexpected column type for column " << getColumnName() << " , expected " + << cspType->type().asCString() << " got " + << ArrowArrayType::TypeClass::type_name() ); + } + } ); } - catch( UnsupportedSwitchType &e ) + catch( UnsupportedSwitchType & e ) { - CSP_THROW( TypeError, "Unexpected column type for column " << getColumnName() << " , expected " << cspType -> type().asCString() << - " got " << ArrowArrayType::TypeClass::type_name() ); + CSP_THROW( TypeError, + "Unexpected column type for column " << getColumnName() << " , expected " + << cspType->type().asCString() << " got " + << ArrowArrayType::TypeClass::type_name() ); } - } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > -void *BaseTypedColumnAdapter::getCurValueUntyped() +template +void * BaseTypedColumnAdapter::getCurValueUntyped() { return &m_curValue; } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > -void -BaseTypedColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> &data ) +template +void BaseTypedColumnAdapter::handleNewBatch( + const std::shared_ptr<::arrow::ChunkedArray> & data ) { - CSP_TRUE_OR_THROW_RUNTIME( data -> num_chunks() == 1, - "Unexpected number of chunks in column" << data -> num_chunks() ); - m_curChunkArray = std::static_pointer_cast( data -> chunk( 0 ) ); + CSP_TRUE_OR_THROW_RUNTIME( data->num_chunks() == 1, "Unexpected number of chunks in column" << data->num_chunks() ); + m_curChunkArray = std::static_pointer_cast( data->chunk( 0 ) ); } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > -void -BaseTypedColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::Array> &data ) +template +void BaseTypedColumnAdapter::handleNewBatch( + const std::shared_ptr<::arrow::Array> & data ) { m_curChunkArray = std::static_pointer_cast( data ); } -ParquetStructAdapter::ParquetStructAdapter( ParquetReader &parquetReader, StructAdapterInfo adapterInfo ) - : m_parquetReader( parquetReader ), - m_structMeta( std::static_pointer_cast( adapterInfo.type() ) -> meta() ) +ParquetStructAdapter::ParquetStructAdapter( ParquetReader & parquetReader, StructAdapterInfo adapterInfo ) + : m_parquetReader( parquetReader ) + , m_structMeta( std::static_pointer_cast( adapterInfo.type() )->meta() ) { - m_resetFunc = [ this, adapterInfo ]() + m_resetFunc = [this, adapterInfo]() { m_fieldSetters.clear(); - for( auto it = adapterInfo.fieldMap() -> begin(); it != adapterInfo.fieldMap() -> end(); ++it ) + for( auto it = adapterInfo.fieldMap()->begin(); it != adapterInfo.fieldMap()->end(); ++it ) { - createFieldSetter( it.value(), *m_parquetReader[ it.key() ] ); + createFieldSetter( it.value(), *m_parquetReader[it.key()] ); } }; m_resetFunc(); - // Note the columns for single struct can come from multiple different readers, so we have to iterate over all columns. - // MultipleFileParquetReader uses the columns from child SingleFileParquetReader readers. - for( auto it = adapterInfo.fieldMap() -> begin(); it != adapterInfo.fieldMap() -> end(); ++it ) + // Note the columns for single struct can come from multiple different readers, so we have to iterate over all + // columns. MultipleFileParquetReader uses the columns from child SingleFileParquetReader readers. + for( auto it = adapterInfo.fieldMap()->begin(); it != adapterInfo.fieldMap()->end(); ++it ) { - auto&& parquetColumnName = it.key(); - auto&& structFieldName = it.value(); - auto& columnReader = m_parquetReader[ parquetColumnName ]->getReader(); - columnReader.addDependentStructAdapter(this); - auto &&fieldPtr = m_structMeta -> field( structFieldName ); - if(fieldPtr -> type() -> type() == CspType::TypeTraits::STRUCT) + auto && parquetColumnName = it.key(); + auto && structFieldName = it.value(); + auto & columnReader = m_parquetReader[parquetColumnName]->getReader(); + columnReader.addDependentStructAdapter( this ); + auto && fieldPtr = m_structMeta->field( structFieldName ); + if( fieldPtr->type()->type() == CspType::TypeTraits::STRUCT ) { - auto &fieldMeta = std::static_pointer_cast( fieldPtr -> type() ) -> meta(); - columnReader.setStructColumnMeta(parquetColumnName, fieldMeta); + auto & fieldMeta = std::static_pointer_cast( fieldPtr->type() )->meta(); + columnReader.setStructColumnMeta( parquetColumnName, fieldMeta ); } } } -ParquetStructAdapter::ParquetStructAdapter( ParquetReader &parquetReader, +ParquetStructAdapter::ParquetStructAdapter( ParquetReader & parquetReader, std::shared_ptr<::arrow::StructType> arrowType, - const std::shared_ptr &structMeta, - const std::vector> &columnAdapters ) - : m_parquetReader( parquetReader ), - m_structMeta( structMeta ) + const std::shared_ptr & structMeta, + const std::vector> & columnAdapters ) + : m_parquetReader( parquetReader ) + , m_structMeta( structMeta ) { - m_resetFunc = []() - { - CSP_THROW( RuntimeException, "Internal error, trying to reset single column struct adapter" ); - }; + m_resetFunc + = []() { CSP_THROW( RuntimeException, "Internal error, trying to reset single column struct adapter" ); }; - auto &arrowColumns = arrowType -> fields(); + auto & arrowColumns = arrowType->fields(); - CSP_TRUE_OR_THROW_RUNTIME( arrowColumns.size() == columnAdapters.size(), "Found mismatch between arrow and csp schema" ); + CSP_TRUE_OR_THROW_RUNTIME( arrowColumns.size() == columnAdapters.size(), + "Found mismatch between arrow and csp schema" ); for( std::size_t i = 0; i < arrowColumns.size(); ++i ) { - auto &arrowCol = *arrowColumns[ i ]; - createFieldSetter( arrowCol.name(), *columnAdapters[ i ] ); + auto & arrowCol = *arrowColumns[i]; + createFieldSetter( arrowCol.name(), *columnAdapters[i] ); } } -void ParquetStructAdapter::createFieldSetter( const std::string &fieldName, - ParquetColumnAdapter &columnAdapter ) +void ParquetStructAdapter::createFieldSetter( const std::string & fieldName, ParquetColumnAdapter & columnAdapter ) { - if(columnAdapter.isMissingColumn()) + if( columnAdapter.isMissingColumn() ) { return; } - auto &&fieldPtr = m_structMeta -> field( fieldName ); - CSP_TRUE_OR_THROW_RUNTIME( fieldPtr != nullptr, - "No field " << fieldName << " in struct " << m_structMeta -> name() ); + auto && fieldPtr = m_structMeta->field( fieldName ); + CSP_TRUE_OR_THROW_RUNTIME( fieldPtr != nullptr, "No field " << fieldName << " in struct " << m_structMeta->name() ); FieldSetter fieldSetter; - if( fieldPtr -> type() -> type() == CspType::TypeTraits::ENUM ) + if( fieldPtr->type()->type() == CspType::TypeTraits::ENUM ) { columnAdapter.ensureType( CspType::STRING() ); - auto enumMetaPtr = std::static_pointer_cast( fieldPtr -> type() ) -> meta(); + auto enumMetaPtr = std::static_pointer_cast( fieldPtr->type() )->meta(); using ColumnType = CspType::Type::toCType::type; - fieldSetter = [ &columnAdapter, fieldPtr, enumMetaPtr ]( StructPtr &s ) + fieldSetter = [&columnAdapter, fieldPtr, enumMetaPtr]( StructPtr & s ) { auto curValue = columnAdapter.getCurValue(); if( curValue.has_value() ) { - fieldPtr -> setValue( s.get(), enumMetaPtr -> fromString( curValue.value().c_str() ) ); + fieldPtr->setValue( s.get(), enumMetaPtr->fromString( curValue.value().c_str() ) ); } }; } else { using TypeSwitch = PrimitiveCspTypeSwitch::Extend; - columnAdapter.ensureType( fieldPtr -> type() ); - if( fieldPtr -> type() -> type() == CspType::Type::STRUCT ) + columnAdapter.ensureType( fieldPtr->type() ); + if( fieldPtr->type()->type() == CspType::Type::STRUCT ) { - auto &fieldMeta = std::static_pointer_cast( fieldPtr -> type() ) -> meta(); - static_cast(columnAdapter).initFromStructMeta( fieldMeta ); + auto & fieldMeta = std::static_pointer_cast( fieldPtr->type() )->meta(); + static_cast( columnAdapter ).initFromStructMeta( fieldMeta ); } if( columnAdapter.isNativeType() ) { fieldSetter = NativeCspTypeSwitch::invoke( - ( columnAdapter.getNativeCspType().get() ), [ &columnAdapter, &fieldPtr ]( auto columnTag ) - { - using ColType = typename decltype(columnTag)::type; - return ConstructibleTypeSwitch::invoke( - fieldPtr -> type().get(), - [ &columnAdapter, &fieldPtr ]( auto tag ) + ( columnAdapter.getNativeCspType().get() ), + [&columnAdapter, &fieldPtr]( auto columnTag ) + { + using ColType = typename decltype( columnTag )::type; + return ConstructibleTypeSwitch::invoke( + fieldPtr->type().get(), + [&columnAdapter, &fieldPtr]( auto tag ) + { + using FieldType = typename decltype( tag )::type; + return FieldSetter( + [&columnAdapter, fieldPtr]( StructPtr & s ) { - using FieldType = typename decltype(tag)::type; - return FieldSetter( [ &columnAdapter, fieldPtr ]( StructPtr &s ) - { - auto curValue = columnAdapter.getCurValue(); - if( curValue.has_value() ) - { - fieldPtr -> setValue( s.get(), csp::cast( - curValue.value() ) ); - } - } ); + auto curValue = columnAdapter.getCurValue(); + if( curValue.has_value() ) + { + fieldPtr->setValue( s.get(), + csp::cast( curValue.value() ) ); + } } ); - } + } ); + } ); } else { - fieldSetter = TypeSwitch::invoke( fieldPtr -> type().get(), [ &columnAdapter, fieldPtr ]( auto tag ) - { - using FieldType = typename decltype(tag)::type; - return FieldSetter( [ &columnAdapter, fieldPtr ]( StructPtr &s ) - { - auto curValue = columnAdapter.getCurValue(); - if( curValue.has_value() ) - { - fieldPtr -> setValue( s.get(), curValue.value() ); - } - } ); - } ); + fieldSetter + = TypeSwitch::invoke( fieldPtr->type().get(), + [&columnAdapter, fieldPtr]( auto tag ) + { + using FieldType = typename decltype( tag )::type; + return FieldSetter( + [&columnAdapter, fieldPtr]( StructPtr & s ) + { + auto curValue = columnAdapter.getCurValue(); + if( curValue.has_value() ) + { + fieldPtr->setValue( s.get(), curValue.value() ); + } + } ); + } ); } } m_fieldSetters.push_back( fieldSetter ); } -void ParquetStructAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol ) +void ParquetStructAdapter::addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol ) { - CSP_TRUE_OR_THROW( inputAdapter -> type() -> type() == CspType::Type::STRUCT, TypeError, - "Subscribing unexpected type " << inputAdapter -> type() -> type() << " as struct for column " ); - auto meta = static_cast( inputAdapter -> type()) -> meta(); + CSP_TRUE_OR_THROW( inputAdapter->type()->type() == CspType::Type::STRUCT, TypeError, + "Subscribing unexpected type " << inputAdapter->type()->type() << " as struct for column " ); + auto meta = static_cast( inputAdapter->type() )->meta(); CSP_TRUE_OR_THROW( meta == m_structMeta, TypeError, - "Subscribing " << meta -> name() << " where " << m_structMeta -> name() << " is expected" ); - - m_valueDispatcher.addSubscriber( [ inputAdapter ]( StructPtr *s ) - { - if( s ) - { - inputAdapter -> pushTick( *s ); - } - else - { - inputAdapter -> pushNullTick(); - } - }, - symbol ); + "Subscribing " << meta->name() << " where " << m_structMeta->name() << " is expected" ); + m_valueDispatcher.addSubscriber( + [inputAdapter]( StructPtr * s ) + { + if( s ) + { + inputAdapter->pushTick( *s ); + } + else + { + inputAdapter->pushNullTick(); + } + }, + symbol ); } -void ParquetStructAdapter::addSubscriber( ValueDispatcher::SubscriberType subscriber, std::optional symbol ) +void ParquetStructAdapter::addSubscriber( ValueDispatcher::SubscriberType subscriber, + std::optional symbol ) { m_valueDispatcher.addSubscriber( subscriber, symbol ); } -void ParquetStructAdapter::dispatchValue( const utils::Symbol *symbol, bool isNull ) +void ParquetStructAdapter::dispatchValue( const utils::Symbol * symbol, bool isNull ) { - if(unlikely(m_needsReset)) + if( unlikely( m_needsReset ) ) { m_resetFunc(); m_needsReset = false; @@ -485,15 +499,15 @@ void ParquetStructAdapter::dispatchValue( const utils::Symbol *symbol, bool isNu } StructPtr s; - StructPtr *dispatchedValue; + StructPtr * dispatchedValue; if( isNull ) { dispatchedValue = nullptr; } else { - s = StructPtr{ m_structMeta -> create() }; - for( auto &fieldSetter : m_fieldSetters ) + s = StructPtr{ m_structMeta->create() }; + for( auto & fieldSetter : m_fieldSetters ) { fieldSetter( s ); } @@ -510,98 +524,98 @@ void ParquetStructAdapter::dispatchValue( const utils::Symbol *symbol, bool isNu } } -template< typename ValueType, typename ArrowArrayType > +template void NativeTypeColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = csp::cast( this -> m_curChunkArray -> Value( curRow ) ); + this->m_curValue = csp::cast( this->m_curChunkArray->Value( curRow ) ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -template< int64_t UNIT > +template void DatetimeColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = csp::DateTime::fromNanoseconds( - m_curChunkArray -> Value( m_parquetReader.getCurRow() ) * UNIT ); + this->m_curValue + = csp::DateTime::fromNanoseconds( m_curChunkArray->Value( m_parquetReader.getCurRow() ) * UNIT ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -template< int64_t UNIT > +template void DurationColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = csp::TimeDelta::fromNanoseconds( - m_curChunkArray -> Value( m_parquetReader.getCurRow() ) * UNIT ); + this->m_curValue + = csp::TimeDelta::fromNanoseconds( m_curChunkArray->Value( m_parquetReader.getCurRow() ) * UNIT ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -template< int64_t UNIT, typename ArrowDateArray > +template void DateColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = csp::DateTime::fromNanoseconds( - this -> m_curChunkArray -> Value( this -> m_parquetReader.getCurRow() ) * UNIT ).date(); + this->m_curValue + = csp::DateTime::fromNanoseconds( this->m_curChunkArray->Value( this->m_parquetReader.getCurRow() ) * UNIT ) + .date(); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -template< int64_t UNIT, typename ArrowTimeArray > +template void TimeColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = csp::Time::fromNanoseconds( - this -> m_curChunkArray -> Value( this -> m_parquetReader.getCurRow() ) * UNIT ); + this->m_curValue + = csp::Time::fromNanoseconds( this->m_curChunkArray->Value( this->m_parquetReader.getCurRow() ) * UNIT ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -void StringColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol ) +void StringColumnAdapter::addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol ) { - if( inputAdapter -> type() -> type() == CspType::TypeTraits::ENUM ) + if( inputAdapter->type()->type() == CspType::TypeTraits::ENUM ) { - auto enumMetaPtr = static_cast( inputAdapter -> type()) -> meta(); - auto callback = std::function( - [ inputAdapter, enumMetaPtr ]( const std::string *val ) + auto enumMetaPtr = static_cast( inputAdapter->type() )->meta(); + auto callback = std::function( + [inputAdapter, enumMetaPtr]( const std::string * val ) + { + if( val ) { - if( val ) - { - inputAdapter -> pushTick( enumMetaPtr -> fromString( val -> c_str() ) ); - } - else - { - inputAdapter -> pushNullTick(); - } - } ); + inputAdapter->pushTick( enumMetaPtr->fromString( val->c_str() ) ); + } + else + { + inputAdapter->pushNullTick(); + } + } ); m_dispatcher.addSubscriber( callback, symbol ); } else @@ -612,73 +626,73 @@ void StringColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, s void StringColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = this -> m_curChunkArray -> GetString( curRow ); + this->m_curValue = this->m_curChunkArray->GetString( curRow ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } void BytesColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = this -> m_curChunkArray -> GetString( curRow ); + this->m_curValue = this->m_curChunkArray->GetString( curRow ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } - void FixedSizeBinaryColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = this -> m_curChunkArray -> GetString( curRow ); + this->m_curValue = this->m_curChunkArray->GetString( curRow ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } void DictionaryColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - auto index = this -> m_curChunkArray -> GetValueIndex( curRow ); - auto dict = static_cast(this -> m_curChunkArray -> dictionary().get()); - this -> m_curValue = dict -> GetString( index ); + auto index = this->m_curChunkArray->GetValueIndex( curRow ); + auto dict = static_cast( this->m_curChunkArray->dictionary().get() ); + this->m_curValue = dict->GetString( index ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } - namespace { template struct ArrayValidValueProvider { template - static T getValue(const V& array, int index) + static T getValue( const V & array, int index ) { - if(!array->IsValid(index)) + if( !array->IsValid( index ) ) { - CSP_THROW(ValueError, "Can't read empty value to array from arrow array of type " << arrow::StringArray::TypeClass::type_name()); + CSP_THROW( ValueError, + "Can't read empty value to array from arrow array of type " + << arrow::StringArray::TypeClass::type_name() ); } - return array->GetView(index); + return array->GetView( index ); } }; @@ -686,207 +700,209 @@ namespace struct ArrayValidValueProvider { template - static double getValue(const V& array, int index) + static double getValue( const V & array, int index ) { - if(!array->IsValid(index)) + if( !array->IsValid( index ) ) { return std::numeric_limits::quiet_NaN(); } - return std::move(array->GetView(index)); + return std::move( array->GetView( index ) ); } }; -} +} // namespace -template< typename ValueArrayType, typename ValueType > -void ListColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol ) +template +void ListColumnAdapter::addSubscriber( ManagedSimInputAdapter * inputAdapter, + std::optional symbol ) { CSP_THROW( NotImplemented, "Trying to subscribe to list column indirectly, via struct field?" ); } -template< typename ValueArrayType, typename ValueType > -void ListColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, - std::optional symbol, - const DialectGenericListReaderInterface::Ptr &listReader ) +template +void ListColumnAdapter::addSubscriber( + ManagedSimInputAdapter * inputAdapter, std::optional symbol, + const DialectGenericListReaderInterface::Ptr & listReader ) { - CSP_TRUE_OR_THROW_RUNTIME( m_listReader == nullptr, - "Trying to subscribe list column in parquet reader more than once, this is not supported" ); + CSP_TRUE_OR_THROW_RUNTIME( + m_listReader == nullptr, + "Trying to subscribe list column in parquet reader more than once, this is not supported" ); CSP_TRUE_OR_THROW_RUNTIME( listReader != nullptr, "Trying to subscribe list column in parquet reader with null listReader" ); Base::addSubscriber( inputAdapter, symbol ); m_listReader = std::dynamic_pointer_cast>( listReader ); CSP_TRUE_OR_THROW_RUNTIME( m_listReader != nullptr, - "Subscribed to parquet column " << getColumnName() << " with type " - << "NumpyArray[" << listReader -> getValueType() -> type().asString() - << "] while " - << " column type in file is NumpyArray[" - << getContainerValueType() -> type().asString() << "]" - << " in file " << m_parquetReader.getCurFileOrTableName() ); + "Subscribed to parquet column " + << getColumnName() << " with type " + << "NumpyArray[" << listReader->getValueType()->type().asString() << "] while " + << " column type in file is NumpyArray[" + << getContainerValueType()->type().asString() << "]" + << " in file " << m_parquetReader.getCurFileOrTableName() ); } -template< typename ValueArrayType, typename ValueType > +template void ListColumnAdapter::readCurValue() { CSP_TRUE_OR_THROW_RUNTIME( m_listReader != nullptr, "Trying to read list value from parquet file but not list reader interface is set" ); - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - auto values = this -> m_curChunkArray -> value_slice( curRow ); + auto values = this->m_curChunkArray->value_slice( curRow ); auto typedValues = std::dynamic_pointer_cast( values ); - auto arrayValue = m_listReader -> create( typedValues -> length() ); - auto* internalBuffer = m_listReader -> getRawDataBuffer( arrayValue ); + auto arrayValue = m_listReader->create( typedValues->length() ); + auto * internalBuffer = m_listReader->getRawDataBuffer( arrayValue ); if( internalBuffer != nullptr ) { - for( int64_t i = 0; i < typedValues -> length(); ++i ) + for( int64_t i = 0; i < typedValues->length(); ++i ) { - *(internalBuffer++) = ArrayValidValueProvider::getValue(typedValues, i); + *( internalBuffer++ ) = ArrayValidValueProvider::getValue( typedValues, i ); } } else { - for( int64_t i = 0; i < typedValues -> length(); ++i ) + for( int64_t i = 0; i < typedValues->length(); ++i ) { - m_listReader -> setValue( arrayValue, i, ArrayValidValueProvider::getValue(typedValues, i) ); + m_listReader->setValue( arrayValue, i, ArrayValidValueProvider::getValue( typedValues, i ) ); } } - this -> m_curValue = std::move( arrayValue ); + this->m_curValue = std::move( arrayValue ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } - template<> void ListColumnAdapter::readCurValue() { CSP_TRUE_OR_THROW_RUNTIME( m_listReader != nullptr, "Trying to read list value from parquet file but not list reader interface is set" ); - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - auto values = this -> m_curChunkArray -> value_slice( curRow ); + auto values = this->m_curChunkArray->value_slice( curRow ); auto typedValues = std::dynamic_pointer_cast( values ); uint32_t maxStringLength = 0; - for( int64_t i = 0; i < typedValues -> length(); ++i ) + for( int64_t i = 0; i < typedValues->length(); ++i ) { - maxStringLength = std::max( ( uint32_t ) ArrayValidValueProvider::getValue(typedValues, i).length(), maxStringLength ); + maxStringLength + = std::max( (uint32_t)ArrayValidValueProvider::getValue( typedValues, i ).length(), + maxStringLength ); } - auto arrayValue = m_listReader -> create( typedValues -> length(), maxStringLength ); + auto arrayValue = m_listReader->create( typedValues->length(), maxStringLength ); - for( int64_t i = 0; i < typedValues -> length(); ++i ) + for( int64_t i = 0; i < typedValues->length(); ++i ) { - m_listReader -> setValue( arrayValue, i, std::string(typedValues -> GetView( i ))); + m_listReader->setValue( arrayValue, i, std::string( typedValues->GetView( i ) ) ); } - this -> m_curValue = std::move( arrayValue ); + this->m_curValue = std::move( arrayValue ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -// We need to force the compiler to instantiate the relevant ListColumnAdapter classes since the implementation is not in header -template -class ListColumnAdapter; +// We need to force the compiler to instantiate the relevant ListColumnAdapter classes since the implementation is not +// in header +template class ListColumnAdapter; -template -class ListColumnAdapter; +template class ListColumnAdapter; -template -class ListColumnAdapter; +template class ListColumnAdapter; -template -class ListColumnAdapter; +template class ListColumnAdapter; -void StructColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol ) +void StructColumnAdapter::addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol ) { - CSP_TRUE_OR_THROW_RUNTIME( inputAdapter -> type() -> type() == CspType::TypeTraits::STRUCT, - "Trying to subscribe with non struct type " << inputAdapter -> type() -> type().asString() ); - auto structMeta = static_cast( inputAdapter -> type()) -> meta(); + CSP_TRUE_OR_THROW_RUNTIME( inputAdapter->type()->type() == CspType::TypeTraits::STRUCT, + "Trying to subscribe with non struct type " << inputAdapter->type()->type().asString() ); + auto structMeta = static_cast( inputAdapter->type() )->meta(); initFromStructMeta( structMeta ); BASE::addSubscriber( inputAdapter, symbol ); } void StructColumnAdapter::readCurValue() { - // TODO: In the future we should make optimization here and skip creating structs for symbols for which there's no subscription - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + // TODO: In the future we should make optimization here and skip creating structs for symbols for which there's no + // subscription + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - for( auto &childColumnAdapter: m_childColumnAdapters ) + for( auto & childColumnAdapter : m_childColumnAdapters ) { - childColumnAdapter -> readCurValue(); + childColumnAdapter->readCurValue(); } - m_structAdapter -> dispatchValue( nullptr ); + m_structAdapter->dispatchValue( nullptr ); } else { - m_structAdapter -> dispatchValue( nullptr, true ); - this -> m_curValue.reset(); + m_structAdapter->dispatchValue( nullptr, true ); + this->m_curValue.reset(); } } -void StructColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> &data ) +void StructColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> & data ) { BASE::handleNewBatch( data ); - auto &childArrays = m_curChunkArray -> fields(); + auto & childArrays = m_curChunkArray->fields(); CSP_TRUE_OR_THROW_RUNTIME( childArrays.size() == m_childColumnAdapters.size(), - "Expected " << m_childColumnAdapters.size() << " child arrays, got " << childArrays.size() ); + "Expected " << m_childColumnAdapters.size() << " child arrays, got " + << childArrays.size() ); for( std::size_t i = 0; i < childArrays.size(); ++i ) { - m_childColumnAdapters[ i ] -> handleNewBatch( childArrays[ i ] ); + m_childColumnAdapters[i]->handleNewBatch( childArrays[i] ); } } -void StructColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::Array> &data ) +void StructColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::Array> & data ) { BASE::handleNewBatch( data ); - auto &childArrays = m_curChunkArray -> fields(); + auto & childArrays = m_curChunkArray->fields(); CSP_TRUE_OR_THROW_RUNTIME( childArrays.size() == m_childColumnAdapters.size(), - "Expected " << m_childColumnAdapters.size() << " child arrays, got " << childArrays.size() ); + "Expected " << m_childColumnAdapters.size() << " child arrays, got " + << childArrays.size() ); for( std::size_t i = 0; i < childArrays.size(); ++i ) { - m_childColumnAdapters[ i ] -> handleNewBatch( childArrays[ i ] ); + m_childColumnAdapters[i]->handleNewBatch( childArrays[i] ); } } -void StructColumnAdapter::initFromStructMeta( const std::shared_ptr &structMeta ) +void StructColumnAdapter::initFromStructMeta( const std::shared_ptr & structMeta ) { if( m_structAdapter ) { - CSP_TRUE_OR_THROW_RUNTIME( m_structAdapter -> getStructMeta() == structMeta, + CSP_TRUE_OR_THROW_RUNTIME( m_structAdapter->getStructMeta() == structMeta, "Trying to subscribe to structure field with struct " - << structMeta -> name() << " and " << m_structAdapter -> getStructMeta() -> name() ); + << structMeta->name() << " and " << m_structAdapter->getStructMeta()->name() ); return; } - m_childColumnAdapters.reserve( m_arrowType -> num_fields() ); - for( auto &field:m_arrowType -> fields() ) + m_childColumnAdapters.reserve( m_arrowType->num_fields() ); + for( auto & field : m_arrowType->fields() ) { m_childColumnAdapters.push_back( createColumnAdapter( m_parquetReader, *field, "" ) ); } - m_structAdapter = std::make_unique( m_parquetReader, - m_arrowType, - structMeta, - m_childColumnAdapters ); - m_structAdapter -> addSubscriber( [ this ]( StructPtr *s ) - { - if( s ) - { - this -> m_curValue = *s; - } - } ); + m_structAdapter + = std::make_unique( m_parquetReader, m_arrowType, structMeta, m_childColumnAdapters ); + m_structAdapter->addSubscriber( + [this]( StructPtr * s ) + { + if( s ) + { + this->m_curValue = *s; + } + } ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetWriter.cpp b/cpp/csp/adapters/parquet/ParquetWriter.cpp index e40d1889..b2a24d19 100644 --- a/cpp/csp/adapters/parquet/ParquetWriter.cpp +++ b/cpp/csp/adapters/parquet/ParquetWriter.cpp @@ -1,57 +1,67 @@ -#include -#include -#include -#include -#include -#include #include #include #include #include +#include +#include +#include +#include +#include #include namespace csp::adapters::parquet { -ParquetWriter::ParquetWriter( ParquetOutputAdapterManager *mgr, std::optional writeTimestampColumn ) - : m_adapterMgr( *mgr ), m_engine( mgr -> engine() ), m_curChunkSize( 0 ), m_writeTimestampColumn( writeTimestampColumn ) -{} +ParquetWriter::ParquetWriter( ParquetOutputAdapterManager * mgr, std::optional writeTimestampColumn ) + : m_adapterMgr( *mgr ) + , m_engine( mgr->engine() ) + , m_curChunkSize( 0 ) + , m_writeTimestampColumn( writeTimestampColumn ) +{ +} -ParquetWriter::ParquetWriter( ParquetOutputAdapterManager *mgr, const Dictionary & properties ) : ParquetWriter( mgr, std::optional{} ) +ParquetWriter::ParquetWriter( ParquetOutputAdapterManager * mgr, const Dictionary & properties ) + : ParquetWriter( mgr, std::optional{} ) { if( properties.exists( "file_metadata" ) ) { auto file_meta = properties.get( "file_metadata" ); m_fileMetaData = std::make_shared(); - for( auto it = file_meta -> begin(); it != file_meta -> end(); ++it ) + for( auto it = file_meta->begin(); it != file_meta->end(); ++it ) { const std::string * value = std::get_if( &it.getUntypedValue() ); if( !value ) CSP_THROW( TypeError, "parquet metadata can only have string values" ); - m_fileMetaData -> Append( it.key(), *value ); + m_fileMetaData->Append( it.key(), *value ); } } if( properties.exists( "column_metadata" ) ) { auto column_metadata = properties.get( "column_metadata" ); - for( auto colIt = column_metadata -> begin(); colIt != column_metadata -> end(); ++colIt ) + for( auto colIt = column_metadata->begin(); colIt != column_metadata->end(); ++colIt ) { const DictionaryPtr * cmeta = std::get_if( &colIt.getUntypedValue() ); if( !cmeta ) - CSP_THROW( TypeError, "parquet column metadata expects dictionary entry per column, got unrecognized type for column '" << colIt.key() << "'" ); + CSP_THROW( + TypeError, + "parquet column metadata expects dictionary entry per column, got unrecognized type for column '" + << colIt.key() << "'" ); auto kv_metadata = std::make_shared(); - for( auto it = (*cmeta) -> begin(); it != (*cmeta) -> end(); ++it ) + for( auto it = ( *cmeta )->begin(); it != ( *cmeta )->end(); ++it ) { const std::string * value = std::get_if( &it.getUntypedValue() ); if( !value ) - CSP_THROW( TypeError, "parquet column metadata can only have string values, got non-string value for metadata on column '" << colIt.key() << "'" ); - kv_metadata -> Append( it.key(), *value ); + CSP_THROW( TypeError, + "parquet column metadata can only have string values, got non-string value for metadata " + "on column '" + << colIt.key() << "'" ); + kv_metadata->Append( it.key(), *value ); } - m_columnMetaData[ colIt.key() ] = kv_metadata; + m_columnMetaData[colIt.key()] = kv_metadata; } } } @@ -64,33 +74,35 @@ ParquetWriter::~ParquetWriter() } } -SingleColumnParquetOutputAdapter *ParquetWriter::getScalarOutputAdapter( CspTypePtr &type, const std::string &columnName ) +SingleColumnParquetOutputAdapter * ParquetWriter::getScalarOutputAdapter( CspTypePtr & type, + const std::string & columnName ) { - auto res = static_cast(getScalarOutputHandler( type, columnName )); + auto res = static_cast( getScalarOutputHandler( type, columnName ) ); return res; } -StructParquetOutputAdapter *ParquetWriter::getStructOutputAdapter( CspTypePtr &type, csp::DictionaryPtr fieldMap ) +StructParquetOutputAdapter * ParquetWriter::getStructOutputAdapter( CspTypePtr & type, csp::DictionaryPtr fieldMap ) { - auto res = static_cast(getStructOutputHandler( type, fieldMap )); + auto res = static_cast( getStructOutputHandler( type, fieldMap ) ); return res; } -SingleColumnParquetOutputHandler *ParquetWriter::getScalarOutputHandler( CspTypePtr &type, const std::string &columnName ) +SingleColumnParquetOutputHandler * ParquetWriter::getScalarOutputHandler( CspTypePtr & type, + const std::string & columnName ) { CSP_TRUE_OR_THROW_RUNTIME( m_publishedColumnNames.emplace( columnName ).second, "Trying to publish column " << columnName << " more than once" ); - //create and register adapter + // create and register adapter auto adapter = createScalarOutputHandler( type, columnName ); m_adapters.emplace_back( adapter ); return adapter; } -StructParquetOutputHandler *ParquetWriter::getStructOutputHandler( CspTypePtr &type, csp::DictionaryPtr fieldMap ) +StructParquetOutputHandler * ParquetWriter::getStructOutputHandler( CspTypePtr & type, csp::DictionaryPtr fieldMap ) { - for( auto it = fieldMap -> begin(); it != fieldMap -> end(); ++it ) + for( auto it = fieldMap->begin(); it != fieldMap->end(); ++it ) { - auto &&columnName = it.value(); + auto && columnName = it.value(); CSP_TRUE_OR_THROW_RUNTIME( m_publishedColumnNames.emplace( columnName ).second, "Trying to publish column " << columnName << " more than once" ); } @@ -101,29 +113,28 @@ StructParquetOutputHandler *ParquetWriter::getStructOutputHandler( CspTypePtr &t return adapter; } -ListColumnParquetOutputHandler *ParquetWriter::getListOutputHandler( CspTypePtr &elemType, const std::string &columnName, - const DialectGenericListWriterInterface::Ptr &listWriterInterface ) +ListColumnParquetOutputHandler * +ParquetWriter::getListOutputHandler( CspTypePtr & elemType, const std::string & columnName, + const DialectGenericListWriterInterface::Ptr & listWriterInterface ) { CSP_TRUE_OR_THROW_RUNTIME( m_publishedColumnNames.emplace( columnName ).second, "Trying to publish column " << columnName << " more than once" ); - //create and register adapter - auto adapter = createListOutputHandler( elemType, columnName, listWriterInterface); + // create and register adapter + auto adapter = createListOutputHandler( elemType, columnName, listWriterInterface ); m_adapters.emplace_back( adapter ); return adapter; } -ListColumnParquetOutputAdapter *ParquetWriter::getListOutputAdapter( - CspTypePtr &elemType, const std::string &columnName, - const DialectGenericListWriterInterface::Ptr &listWriterInterface ) +ListColumnParquetOutputAdapter * +ParquetWriter::getListOutputAdapter( CspTypePtr & elemType, const std::string & columnName, + const DialectGenericListWriterInterface::Ptr & listWriterInterface ) { - auto res = static_cast(getListOutputHandler( elemType, columnName, listWriterInterface )); + auto res = static_cast( + getListOutputHandler( elemType, columnName, listWriterInterface ) ); return res; } -PushInputAdapter *ParquetWriter::getStatusAdapter() -{ - return nullptr; -} +PushInputAdapter * ParquetWriter::getStatusAdapter() { return nullptr; } void ParquetWriter::start() { @@ -131,43 +142,44 @@ void ParquetWriter::start() if( !m_writeTimestampColumn.has_value() && !m_adapterMgr.getTimestampColumnName().empty() ) { m_writeTimestampColumn = true; - m_columnBuilders.push_back( std::make_shared( m_adapterMgr.getTimestampColumnName(), getChunkSize() ) ); + m_columnBuilders.push_back( + std::make_shared( m_adapterMgr.getTimestampColumnName(), getChunkSize() ) ); std::shared_ptr colMetaData; auto colMetaIt = m_columnMetaData.find( m_adapterMgr.getTimestampColumnName() ); if( colMetaIt != m_columnMetaData.end() ) { - colMetaData = colMetaIt -> second; + colMetaData = colMetaIt->second; m_columnMetaData.erase( colMetaIt ); } - arrowFields.push_back( - arrow::field( m_adapterMgr.getTimestampColumnName(), m_columnBuilders.back() -> getDataType(), colMetaData ) ); + arrowFields.push_back( arrow::field( m_adapterMgr.getTimestampColumnName(), + m_columnBuilders.back()->getDataType(), colMetaData ) ); } else { m_writeTimestampColumn = false; } - for( auto &&adapter:m_adapters ) + for( auto && adapter : m_adapters ) { - for( unsigned i = 0; i < adapter -> getNumColumns(); ++i ) + for( unsigned i = 0; i < adapter->getNumColumns(); ++i ) { - m_columnBuilders.push_back( adapter -> getColumnArrayBuilder( i ) ); + m_columnBuilders.push_back( adapter->getColumnArrayBuilder( i ) ); std::shared_ptr colMetaData; - auto colMetaIt = m_columnMetaData.find( m_columnBuilders.back() -> getColumnName() ); + auto colMetaIt = m_columnMetaData.find( m_columnBuilders.back()->getColumnName() ); if( colMetaIt != m_columnMetaData.end() ) { - colMetaData = colMetaIt -> second; + colMetaData = colMetaIt->second; m_columnMetaData.erase( colMetaIt ); } - arrowFields.push_back( arrow::field( m_columnBuilders.back() -> getColumnName(), - m_columnBuilders.back() -> getDataType(), - colMetaData ) ); + arrowFields.push_back( arrow::field( m_columnBuilders.back()->getColumnName(), + m_columnBuilders.back()->getDataType(), colMetaData ) ); } } if( !m_columnMetaData.empty() ) - CSP_THROW( ValueError, "parquet column metadata has unmapped column: '" << m_columnMetaData.begin() -> first << "'" ); + CSP_THROW( ValueError, + "parquet column metadata has unmapped column: '" << m_columnMetaData.begin()->first << "'" ); initFileWriterContainer( arrow::schema( arrowFields, m_fileMetaData ) ); } @@ -179,7 +191,7 @@ void ParquetWriter::stop() { writeCurChunkToFile(); } - m_fileWriterWrapperContainer -> close(); + m_fileWriterWrapperContainer->close(); m_fileWriterWrapperContainer = nullptr; } } @@ -194,12 +206,12 @@ void ParquetWriter::onEndCycle() if( m_writeTimestampColumn.value() ) { // Set the timestamp value it's always the first - now = m_adapterMgr.rootEngine() -> now(); - static_cast(m_columnBuilders[ 0 ].get()) -> setValue( now ); + now = m_adapterMgr.rootEngine()->now(); + static_cast( m_columnBuilders[0].get() )->setValue( now ); } - for( auto &&columnBuilder:m_columnBuilders ) + for( auto && columnBuilder : m_columnBuilders ) { - columnBuilder -> handleRowFinished(); + columnBuilder->handleRowFinished(); } if( ++m_curChunkSize >= getChunkSize() ) { @@ -210,55 +222,55 @@ void ParquetWriter::onEndCycle() bool ParquetWriter::isFileOpen() const { - return m_fileWriterWrapperContainer != nullptr && m_fileWriterWrapperContainer -> isOpen(); + return m_fileWriterWrapperContainer != nullptr && m_fileWriterWrapperContainer->isOpen(); } -void ParquetWriter::onFileNameChange( const std::string &fileName ) +void ParquetWriter::onFileNameChange( const std::string & fileName ) { - CSP_TRUE_OR_THROW_RUNTIME( m_fileWriterWrapperContainer, "Trying to set file name when file writer already closed" ); + CSP_TRUE_OR_THROW_RUNTIME( m_fileWriterWrapperContainer, + "Trying to set file name when file writer already closed" ); writeCurChunkToFile(); - m_fileWriterWrapperContainer -> close(); + m_fileWriterWrapperContainer->close(); if( !fileName.empty() ) { - m_fileWriterWrapperContainer - -> open( fileName, m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); + m_fileWriterWrapperContainer->open( fileName, m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); } } -SingleColumnParquetOutputHandler *ParquetWriter::createScalarOutputHandler( CspTypePtr type, const std::string &name ) +SingleColumnParquetOutputHandler * ParquetWriter::createScalarOutputHandler( CspTypePtr type, const std::string & name ) { - return m_engine -> createOwnedObject( *this, type, name ); + return m_engine->createOwnedObject( *this, type, name ); } -ListColumnParquetOutputHandler *ParquetWriter::createListOutputHandler( CspTypePtr &elemType, const std::string &columnName, - DialectGenericListWriterInterface::Ptr listWriterInterface ) +ListColumnParquetOutputHandler * +ParquetWriter::createListOutputHandler( CspTypePtr & elemType, const std::string & columnName, + DialectGenericListWriterInterface::Ptr listWriterInterface ) { - return m_engine -> createOwnedObject( *this, elemType, columnName, listWriterInterface ); + return m_engine->createOwnedObject( *this, elemType, columnName, + listWriterInterface ); } - -StructParquetOutputHandler *ParquetWriter::createStructOutputHandler( CspTypePtr type, - const DictionaryPtr &fieldMap ) +StructParquetOutputHandler * ParquetWriter::createStructOutputHandler( CspTypePtr type, const DictionaryPtr & fieldMap ) { - return m_engine -> createOwnedObject( *this, type, fieldMap ); + return m_engine->createOwnedObject( *this, type, fieldMap ); } void ParquetWriter::initFileWriterContainer( std::shared_ptr schema ) { if( m_adapterMgr.isSplitColumnsToFiles() ) { - m_fileWriterWrapperContainer = std::make_unique( schema, - m_adapterMgr.isWriteArrowBinary() ); + m_fileWriterWrapperContainer + = std::make_unique( schema, m_adapterMgr.isWriteArrowBinary() ); } else { - m_fileWriterWrapperContainer = std::make_unique( schema, - m_adapterMgr.isWriteArrowBinary() ); + m_fileWriterWrapperContainer + = std::make_unique( schema, m_adapterMgr.isWriteArrowBinary() ); } if( !m_adapterMgr.getFileName().empty() ) { - m_fileWriterWrapperContainer -> open( m_adapterMgr.getFileName(), - m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); + m_fileWriterWrapperContainer->open( m_adapterMgr.getFileName(), m_adapterMgr.getCompression(), + m_adapterMgr.isAllowOverwrite() ); } } @@ -268,16 +280,16 @@ void ParquetWriter::writeCurChunkToFile() { if( unlikely( !isFileOpen() ) ) { - if(m_curChunkSize != 0 ) + if( m_curChunkSize != 0 ) { - CSP_THROW( csp::RuntimeException, "Trying to write to parquet/arrow file, when no file name was provided" ); - + CSP_THROW( csp::RuntimeException, + "Trying to write to parquet/arrow file, when no file name was provided" ); } return; } - m_fileWriterWrapperContainer -> writeData( m_columnBuilders ); + m_fileWriterWrapperContainer->writeData( m_columnBuilders ); m_curChunkSize = 0; } } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/utils/JSONMessageStructConverter.cpp b/cpp/csp/adapters/utils/JSONMessageStructConverter.cpp index 8e1217c6..4d5c469f 100644 --- a/cpp/csp/adapters/utils/JSONMessageStructConverter.cpp +++ b/cpp/csp/adapters/utils/JSONMessageStructConverter.cpp @@ -7,26 +7,15 @@ namespace csp::adapters::utils { -using SupportedCspTypeSwitch = PartialSwitchCspType; - -using SupportedArrayCspTypeSwitch = PartialSwitchCspType; +using SupportedCspTypeSwitch + = PartialSwitchCspType; +using SupportedArrayCspTypeSwitch + = PartialSwitchCspType; template<> bool JSONMessageStructConverter::convertJSON( const char * fieldname, const rapidjson::Value & jValue, bool * ) @@ -65,7 +54,8 @@ double JSONMessageStructConverter::convertJSON( const char * fieldname, const ra } template<> -std::string JSONMessageStructConverter::convertJSON( const char * fieldname, const rapidjson::Value & jValue, std::string * ) +std::string JSONMessageStructConverter::convertJSON( const char * fieldname, const rapidjson::Value & jValue, + std::string * ) { if( jValue.IsString() ) return jValue.GetString(); @@ -82,10 +72,18 @@ DateTime JSONMessageStructConverter::convertJSON( const char * fieldname, const DateTime dt; switch( m_datetimeType ) { - case DateTimeWireType::UINT64_NANOS: dt = DateTime::fromNanoseconds( raw ); break; - case DateTimeWireType::UINT64_MICROS: dt = DateTime::fromMicroseconds( raw ); break; - case DateTimeWireType::UINT64_MILLIS: dt = DateTime::fromMilliseconds( raw ); break; - case DateTimeWireType::UINT64_SECONDS: dt = DateTime::fromSeconds( raw ); break; + case DateTimeWireType::UINT64_NANOS: + dt = DateTime::fromNanoseconds( raw ); + break; + case DateTimeWireType::UINT64_MICROS: + dt = DateTime::fromMicroseconds( raw ); + break; + case DateTimeWireType::UINT64_MILLIS: + dt = DateTime::fromMilliseconds( raw ); + break; + case DateTimeWireType::UINT64_SECONDS: + dt = DateTime::fromSeconds( raw ); + break; case DateTimeWireType::UNKNOWN: case DateTimeWireType::NUM_TYPES: @@ -99,39 +97,44 @@ DateTime JSONMessageStructConverter::convertJSON( const char * fieldname, const } template<> -CspEnum JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType & type, const FieldEntry &, const rapidjson::Value & jValue, CspEnum * ) +CspEnum JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType & type, const FieldEntry &, + const rapidjson::Value & jValue, CspEnum * ) { if( !jValue.IsString() ) CSP_THROW( TypeError, "expected ENUM type for json field " << fieldname ); auto & cspEnumType = static_cast( type ); - return cspEnumType.meta() -> fromString( jValue.GetString() ); + return cspEnumType.meta()->fromString( jValue.GetString() ); } template<> -StructPtr JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType &, const FieldEntry & entry, const rapidjson::Value & jValue, StructPtr * ) +StructPtr JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType &, const FieldEntry & entry, + const rapidjson::Value & jValue, StructPtr * ) { if( !jValue.IsObject() ) CSP_THROW( TypeError, "expected Nested object type for json field " << fieldname ); - const CspStructType & sType = static_cast( *entry.sField -> type() ); - auto struct_ = sType.meta() -> create(); - auto & fields = *entry.nestedFields; + const CspStructType & sType = static_cast( *entry.sField->type() ); + auto struct_ = sType.meta()->create(); + auto & fields = *entry.nestedFields; for( auto jit = jValue.MemberBegin(); jit != jValue.MemberEnd(); ++jit ) { - auto sIt = fields.find( jit -> name.GetString() ); + auto sIt = fields.find( jit->name.GetString() ); if( sIt == fields.end() ) continue; - auto & nestedEntry = sIt -> second; + auto & nestedEntry = sIt->second; SupportedCspTypeSwitch::invoke( - nestedEntry.sField -> type().get(), [this,&jit,&nestedEntry,&struct_]( auto tag ) + nestedEntry.sField->type().get(), + [this, &jit, &nestedEntry, &struct_]( auto tag ) { - using T = typename decltype(tag)::type; - auto & jValue = jit -> value; + using T = typename decltype( tag )::type; + auto & jValue = jit->value; - nestedEntry.sField -> setValue( struct_.get(), convertJSON( jit -> name.GetString(), *nestedEntry.sField -> type(), nestedEntry, jValue, static_cast( nullptr ) ) ); + nestedEntry.sField->setValue( struct_.get(), + convertJSON( jit->name.GetString(), *nestedEntry.sField->type(), + nestedEntry, jValue, static_cast( nullptr ) ) ); } ); } @@ -139,7 +142,9 @@ StructPtr JSONMessageStructConverter::convertJSON( const char * fieldname, const } template -std::vector JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType & type, const FieldEntry &, const rapidjson::Value & jValue, std::vector * x ) +std::vector JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType & type, + const FieldEntry &, const rapidjson::Value & jValue, + std::vector * x ) { if( !jValue.IsArray() ) CSP_THROW( TypeError, "expected ARRAY type for json field " << fieldname ); @@ -153,31 +158,32 @@ std::vector JSONMessageStructConverter::convertJSON( const char * fiel out.reserve( jArray.Size() ); for( auto & v : jArray ) { - //note that we dont pass FieldEntry to convert here, this doesnt support arrays of structs - out.emplace_back( convertJSON( fieldname, elemType, {}, v, ( ElemT * ) nullptr) ); + // note that we dont pass FieldEntry to convert here, this doesnt support arrays of structs + out.emplace_back( convertJSON( fieldname, elemType, {}, v, (ElemT *)nullptr ) ); } return out; } -JSONMessageStructConverter::JSONMessageStructConverter( const CspTypePtr & type, - const Dictionary & properties ) : MessageStructConverter( type, properties ) +JSONMessageStructConverter::JSONMessageStructConverter( const CspTypePtr & type, const Dictionary & properties ) + : MessageStructConverter( type, properties ) { - if( type -> type() != CspType::Type::STRUCT ) - CSP_THROW( TypeError, "JSONMessageStructConverter expects type struct got " << type -> type() ); + if( type->type() != CspType::Type::STRUCT ) + CSP_THROW( TypeError, "JSONMessageStructConverter expects type struct got " << type->type() ); const Dictionary & fieldMap = *properties.get( "field_map" ); - m_datetimeType = DateTimeWireType( properties.get( "datetime_type" ) ); - m_fields = buildFields( static_cast( *type ), fieldMap ); + m_datetimeType = DateTimeWireType( properties.get( "datetime_type" ) ); + m_fields = buildFields( static_cast( *type ), fieldMap ); } -JSONMessageStructConverter::Fields JSONMessageStructConverter::buildFields( const CspStructType & type, const Dictionary & fieldMap ) +JSONMessageStructConverter::Fields JSONMessageStructConverter::buildFields( const CspStructType & type, + const Dictionary & fieldMap ) { JSONMessageStructConverter::Fields out; for( auto it = fieldMap.begin(); it != fieldMap.end(); ++it ) { - auto & fieldName = it.key(); + auto & fieldName = it.key(); std::string structField; DictionaryPtr nestedFieldMap; @@ -187,27 +193,35 @@ JSONMessageStructConverter::Fields JSONMessageStructConverter::buildFields( cons else { if( !it.hasValue() ) - CSP_THROW( TypeError, "fieldMap expected string or dict for field " << fieldName << " on struct " << type.meta() -> name() ); + CSP_THROW( TypeError, + "fieldMap expected string or dict for field " << fieldName << " on struct " + << type.meta()->name() ); auto nestedDict = it.value(); - if( nestedDict -> size() != 1 ) - CSP_THROW( ValueError, "Expected nested fieldmap for incoming JSON field " << fieldName << " to have a single key : map entry" ); - structField = nestedDict -> begin().key(); - nestedFieldMap = nestedDict -> begin().value(); + if( nestedDict->size() != 1 ) + CSP_THROW( ValueError, + "Expected nested fieldmap for incoming JSON field " << fieldName + << " to have a single key : map entry" ); + structField = nestedDict->begin().key(); + nestedFieldMap = nestedDict->begin().value(); } - auto sField = type.meta() -> field( structField ); + auto sField = type.meta()->field( structField ); if( !sField ) - CSP_THROW( ValueError, "field " << structField << " is not a valid field on struct type " << type.meta() -> name() ); + CSP_THROW( ValueError, + "field " << structField << " is not a valid field on struct type " << type.meta()->name() ); std::shared_ptr nestedFields; - if( sField -> type() -> type() == CspType::Type::STRUCT ) + if( sField->type()->type() == CspType::Type::STRUCT ) { if( !nestedFieldMap ) - CSP_THROW( ValueError, "invalid field_map entry for nested struct field " << sField -> fieldname() << " on struct type " << type.meta() -> name() ); - nestedFields = std::make_shared( buildFields( static_cast( *sField -> type() ), *nestedFieldMap ) ); + CSP_THROW( ValueError, + "invalid field_map entry for nested struct field " + << sField->fieldname() << " on struct type " << type.meta()->name() ); + nestedFields = std::make_shared( + buildFields( static_cast( *sField->type() ), *nestedFieldMap ) ); } - //keep strings around to keep const char * keys alive + // keep strings around to keep const char * keys alive m_jsonkeys.emplace_back( fieldName ); out.emplace( m_jsonkeys.back().c_str(), FieldEntry{ sField, nestedFields } ); } @@ -216,32 +230,36 @@ JSONMessageStructConverter::Fields JSONMessageStructConverter::buildFields( cons csp::StructPtr JSONMessageStructConverter::asStruct( void * bytes, size_t size ) { - const char * rawmsg = (const char *) bytes; + const char * rawmsg = (const char *)bytes; rapidjson::Document document; rapidjson::ParseResult ok = document.Parse( rawmsg, size ); if( !ok ) - CSP_THROW( ValueError, "Failed to parse message as JSON: " << rapidjson::GetParseError_En( ok.Code() ) << " on msg: " << std::string( rawmsg, size ) ); + CSP_THROW( ValueError, + "Failed to parse message as JSON: " << rapidjson::GetParseError_En( ok.Code() ) + << " on msg: " << std::string( rawmsg, size ) ); - StructPtr data = m_structMeta -> create(); + StructPtr data = m_structMeta->create(); for( auto jit = document.MemberBegin(); jit != document.MemberEnd(); ++jit ) { - auto sIt = m_fields.find( jit -> name.GetString() ); + auto sIt = m_fields.find( jit->name.GetString() ); if( sIt == m_fields.end() ) continue; - auto & entry = sIt -> second; + auto & entry = sIt->second; SupportedCspTypeSwitch::invoke( - entry.sField -> type().get(), [this,&jit,&entry,&data]( auto tag ) + entry.sField->type().get(), + [this, &jit, &entry, &data]( auto tag ) { - using T = typename decltype(tag)::type; - auto & jValue = jit -> value; - entry.sField -> setValue( data.get(), convertJSON( jit -> name.GetString(), *entry.sField -> type(), entry, jValue, static_cast( nullptr ) ) ); - } - ); + using T = typename decltype( tag )::type; + auto & jValue = jit->value; + entry.sField->setValue( data.get(), + convertJSON( jit->name.GetString(), *entry.sField->type(), entry, jValue, + static_cast( nullptr ) ) ); + } ); } return data; } -} +} // namespace csp::adapters::utils diff --git a/cpp/csp/adapters/utils/MessageEnums.cpp b/cpp/csp/adapters/utils/MessageEnums.cpp index 05415fe1..4c281be1 100644 --- a/cpp/csp/adapters/utils/MessageEnums.cpp +++ b/cpp/csp/adapters/utils/MessageEnums.cpp @@ -3,19 +3,9 @@ namespace csp { -INIT_CSP_ENUM( csp::adapters::utils::DateTimeWireType, - "UNKNOWN", - "UINT64_NANOS", - "UINT64_MICROS", - "UINT64_MILLIS", - "UINT64_SECONDS" -); +INIT_CSP_ENUM( csp::adapters::utils::DateTimeWireType, "UNKNOWN", "UINT64_NANOS", "UINT64_MICROS", "UINT64_MILLIS", + "UINT64_SECONDS" ); -INIT_CSP_ENUM( csp::adapters::utils::MsgProtocol, - "UNKNOWN", - "JSON", - "PROTOBUF", - "RAW_BYTES" -); +INIT_CSP_ENUM( csp::adapters::utils::MsgProtocol, "UNKNOWN", "JSON", "PROTOBUF", "RAW_BYTES" ); -} +} // namespace csp diff --git a/cpp/csp/adapters/utils/MessageStructConverter.cpp b/cpp/csp/adapters/utils/MessageStructConverter.cpp index 80e1c3d8..255218d7 100644 --- a/cpp/csp/adapters/utils/MessageStructConverter.cpp +++ b/cpp/csp/adapters/utils/MessageStructConverter.cpp @@ -1,30 +1,32 @@ -#include #include +#include #include #include namespace csp::adapters::utils { -MessageStructConverter::MessageStructConverter( const CspTypePtr & type, const Dictionary & properties ) : m_type( type ) +MessageStructConverter::MessageStructConverter( const CspTypePtr & type, const Dictionary & properties ) + : m_type( type ) { - if( type -> type() == CspType::Type::STRUCT ) - m_structMeta = std::static_pointer_cast( type ) -> meta(); + if( type->type() == CspType::Type::STRUCT ) + m_structMeta = std::static_pointer_cast( type )->meta(); } MessageStructConverterCache::MessageStructConverterCache() { registerConverter( MsgProtocol::RAW_BYTES, &RawBytesMessageStructConverter::create ); - registerConverter( MsgProtocol::JSON, &JSONMessageStructConverter::create ); - registerConverter( MsgProtocol::PROTOBUF, &ProtobufMessageStructConverter::create ); + registerConverter( MsgProtocol::JSON, &JSONMessageStructConverter::create ); + registerConverter( MsgProtocol::PROTOBUF, &ProtobufMessageStructConverter::create ); } bool MessageStructConverterCache::registerConverter( MsgProtocol protocol, Creator creator ) { - if( m_creators[ protocol ] ) - CSP_THROW( RuntimeException, "Attempted to register creator for MessageStructConverter type " << protocol << " more than once" ); + if( m_creators[protocol] ) + CSP_THROW( RuntimeException, + "Attempted to register creator for MessageStructConverter type " << protocol << " more than once" ); - m_creators[ protocol ] = creator; + m_creators[protocol] = creator; return true; } @@ -40,16 +42,16 @@ MessageStructConverterPtr MessageStructConverterCache::create( const CspTypePtr auto rv = m_cache.emplace( CacheKey{ type.get(), properties }, nullptr ); if( !rv.second ) - return rv.first -> second; + return rv.first->second; auto protocol = MsgProtocol( properties.get( "protocol" ) ); - auto creator = m_creators[ protocol ]; + auto creator = m_creators[protocol]; if( !creator ) CSP_THROW( ValueError, "MessageStructConverter for type " << protocol << " is not defined" ); - auto result = std::shared_ptr( creator( type, properties ) ); - rv.first -> second = result; - return rv.first -> second; + auto result = std::shared_ptr( creator( type, properties ) ); + rv.first->second = result; + return rv.first->second; } -} +} // namespace csp::adapters::utils diff --git a/cpp/csp/adapters/utils/MessageWriter.cpp b/cpp/csp/adapters/utils/MessageWriter.cpp index 8d8c6296..df77a0a4 100644 --- a/cpp/csp/adapters/utils/MessageWriter.cpp +++ b/cpp/csp/adapters/utils/MessageWriter.cpp @@ -1,20 +1,19 @@ -#include #include +#include namespace csp::adapters::utils { -MessageWriter::~MessageWriter() -{ -} +MessageWriter::~MessageWriter() {} -OutputDataMapper::OutputDataMapper( const CspTypePtr & type, const Dictionary & fieldMap ) : m_type( type ) +OutputDataMapper::OutputDataMapper( const CspTypePtr & type, const Dictionary & fieldMap ) + : m_type( type ) { m_hasFields = !fieldMap.empty(); - bool isStruct = type -> type() == CspType::Type::STRUCT; + bool isStruct = type->type() == CspType::Type::STRUCT; if( isStruct ) - m_structMeta = std::static_pointer_cast( type ) -> meta(); + m_structMeta = std::static_pointer_cast( type )->meta(); if( isStruct ) m_messageFields = populateStructFields( m_structMeta, fieldMap ); @@ -27,43 +26,55 @@ OutputDataMapper::OutputDataMapper( const CspTypePtr & type, const Dictionary & } } -OutputDataMapper::Fields OutputDataMapper::populateStructFields( const StructMetaPtr & structMeta, const Dictionary & fieldMap ) +OutputDataMapper::Fields OutputDataMapper::populateStructFields( const StructMetaPtr & structMeta, + const Dictionary & fieldMap ) { Fields out; for( auto it = fieldMap.begin(); it != fieldMap.end(); ++it ) { auto & structField = it.key(); - auto sField = structMeta -> field( structField ); + auto sField = structMeta->field( structField ); if( !sField ) - CSP_THROW( ValueError, "field " << structField << " is not a valid field on struct type " << structMeta -> name() ); + CSP_THROW( ValueError, + "field " << structField << " is not a valid field on struct type " << structMeta->name() ); if( it.hasValue() ) { - if( sField -> type() -> type() == CspType::Type::STRUCT ) - CSP_THROW( TypeError, "Expected non-struct type for fieldmap on struct field " << structField << " on struct " << structMeta -> name() ); + if( sField->type()->type() == CspType::Type::STRUCT ) + CSP_THROW( TypeError, + "Expected non-struct type for fieldmap on struct field " << structField << " on struct " + << structMeta->name() ); - auto & fieldName = it.value(); + auto & fieldName = it.value(); out.emplace_back( FieldEntry{ fieldName, sField, {} } ); } else { if( !it.hasValue() ) - CSP_THROW( TypeError, "fieldMap expected string or dict for field \"" << structField << "\" on struct \"" << structMeta -> name() << "\""); + CSP_THROW( TypeError, + "fieldMap expected string or dict for field \"" << structField << "\" on struct \"" + << structMeta->name() << "\"" ); auto nestedEntry = it.value(); - if( nestedEntry -> size() != 1 || !nestedEntry -> begin().hasValue() ) - CSP_THROW( ValueError, "Expected nested fieldmap for outgoing field \"" << structField << "\" on struct \"" << structMeta -> name() << - "\" to have a dict of single key : nested_fieldmap entry" ); - - if( sField -> type() -> type() != CspType::Type::STRUCT ) - CSP_THROW( TypeError, "Expected structed type for nested struct field \"" << structField << "\" on struct \"" << structMeta -> name() << "\" got: \"" << sField -> type() -> type() << "\"" ); - - //sub-struct - auto nestedStructType = std::static_pointer_cast( sField -> type() ); - auto fieldName = nestedEntry -> begin().key(); - auto nestedFieldMap = nestedEntry -> begin().value(); - - auto nestedFields = std::make_shared( populateStructFields( nestedStructType -> meta(), *nestedFieldMap ) ); + if( nestedEntry->size() != 1 || !nestedEntry->begin().hasValue() ) + CSP_THROW( ValueError, + "Expected nested fieldmap for outgoing field \"" + << structField << "\" on struct \"" << structMeta->name() + << "\" to have a dict of single key : nested_fieldmap entry" ); + + if( sField->type()->type() != CspType::Type::STRUCT ) + CSP_THROW( TypeError, + "Expected structed type for nested struct field \"" << structField << "\" on struct \"" + << structMeta->name() << "\" got: \"" + << sField->type()->type() << "\"" ); + + // sub-struct + auto nestedStructType = std::static_pointer_cast( sField->type() ); + auto fieldName = nestedEntry->begin().key(); + auto nestedFieldMap = nestedEntry->begin().value(); + + auto nestedFields + = std::make_shared( populateStructFields( nestedStructType->meta(), *nestedFieldMap ) ); out.emplace_back( FieldEntry{ fieldName, sField, nestedFields } ); } } @@ -71,9 +82,7 @@ OutputDataMapper::Fields OutputDataMapper::populateStructFields( const StructMet return out; } -OutputDataMapperCache::OutputDataMapperCache() -{ -} +OutputDataMapperCache::OutputDataMapperCache() {} OutputDataMapperCache & OutputDataMapperCache::instance() { @@ -87,11 +96,11 @@ OutputDataMapperPtr OutputDataMapperCache::create( const CspTypePtr & type, cons auto rv = m_cache.emplace( CacheKey{ type.get(), fieldMap }, nullptr ); if( !rv.second ) - return rv.first -> second; + return rv.first->second; - auto helper = std::make_shared( type, fieldMap ); - rv.first -> second = helper; - return rv.first -> second; + auto helper = std::make_shared( type, fieldMap ); + rv.first->second = helper; + return rv.first->second; } -} +} // namespace csp::adapters::utils diff --git a/cpp/csp/adapters/utils/ProtobufHelper.cpp b/cpp/csp/adapters/utils/ProtobufHelper.cpp index f1e1600b..856dd904 100644 --- a/cpp/csp/adapters/utils/ProtobufHelper.cpp +++ b/cpp/csp/adapters/utils/ProtobufHelper.cpp @@ -9,35 +9,32 @@ namespace proto = google::protobuf; namespace csp::adapters::utils { -//For some reason the default proto::Importer does not import well know types, so we had to write -//our own version +// For some reason the default proto::Importer does not import well know types, so we had to write +// our own version class ProtoImporterAux : public proto::compiler::MultiFileErrorCollector { public: - ProtoImporterAux( proto::compiler::SourceTree *source_tree ) - : m_wellKnownTypesDatabase( *proto::DescriptorPool::generated_pool() ), - m_database( source_tree, &m_wellKnownTypesDatabase ), - m_pool( &m_database, m_database.GetValidationErrorCollector()) + ProtoImporterAux( proto::compiler::SourceTree * source_tree ) + : m_wellKnownTypesDatabase( *proto::DescriptorPool::generated_pool() ) + , m_database( source_tree, &m_wellKnownTypesDatabase ) + , m_pool( &m_database, m_database.GetValidationErrorCollector() ) { m_pool.EnforceWeakDependencies( true ); m_database.RecordErrorsTo( this ); } - const proto::FileDescriptor *Import( const std::string & filename ) - { - return m_pool.FindFileByName( filename ); - } + const proto::FileDescriptor * Import( const std::string & filename ) { return m_pool.FindFileByName( filename ); } - void AddError( const std::string& filename, int line, int column, - const std::string& message ) override + void AddError( const std::string & filename, int line, int column, const std::string & message ) override { - CSP_THROW( RuntimeException, "Failed to load proto schema " << filename << ":" << line << ":" << column << ": " << message ); + CSP_THROW( RuntimeException, + "Failed to load proto schema " << filename << ":" << line << ":" << column << ": " << message ); } - + private: - proto::DescriptorPoolDatabase m_wellKnownTypesDatabase; + proto::DescriptorPoolDatabase m_wellKnownTypesDatabase; proto::compiler::SourceTreeDescriptorDatabase m_database; - proto::DescriptorPool m_pool; + proto::DescriptorPool m_pool; }; ProtobufHelper::ProtobufHelper() @@ -57,10 +54,10 @@ const proto::FileDescriptor * ProtobufHelper::import( const std::string & schema } } - return m_protoImporter -> Import( filename ); + return m_protoImporter->Import( filename ); } -const proto::Message * ProtobufHelper::getMessage( const proto::Descriptor* type ) +const proto::Message * ProtobufHelper::getMessage( const proto::Descriptor * type ) { return m_protoMsgFactory.GetPrototype( type ); } @@ -75,16 +72,24 @@ proto::FieldDescriptor::CppType ProtobufHelper::cspToProtoCppType( const CspType { switch( type.type() ) { - //NOTE if new types are added make sure parsing code switch statement below is updated - case CspType::Type::BOOL: return proto::FieldDescriptor::CPPTYPE_BOOL; - case CspType::Type::INT32: return proto::FieldDescriptor::CPPTYPE_INT32; - case CspType::Type::UINT32: return proto::FieldDescriptor::CPPTYPE_UINT32; - case CspType::Type::INT64: return proto::FieldDescriptor::CPPTYPE_INT64; - case CspType::Type::UINT64: return proto::FieldDescriptor::CPPTYPE_UINT64; - case CspType::Type::DOUBLE: return proto::FieldDescriptor::CPPTYPE_DOUBLE; - case CspType::Type::STRING: return proto::FieldDescriptor::CPPTYPE_STRING; - case CspType::Type::STRUCT: return proto::FieldDescriptor::CPPTYPE_MESSAGE; - case CspType::Type::ARRAY: + // NOTE if new types are added make sure parsing code switch statement below is updated + case CspType::Type::BOOL: + return proto::FieldDescriptor::CPPTYPE_BOOL; + case CspType::Type::INT32: + return proto::FieldDescriptor::CPPTYPE_INT32; + case CspType::Type::UINT32: + return proto::FieldDescriptor::CPPTYPE_UINT32; + case CspType::Type::INT64: + return proto::FieldDescriptor::CPPTYPE_INT64; + case CspType::Type::UINT64: + return proto::FieldDescriptor::CPPTYPE_UINT64; + case CspType::Type::DOUBLE: + return proto::FieldDescriptor::CPPTYPE_DOUBLE; + case CspType::Type::STRING: + return proto::FieldDescriptor::CPPTYPE_STRING; + case CspType::Type::STRUCT: + return proto::FieldDescriptor::CPPTYPE_MESSAGE; + case CspType::Type::ARRAY: { auto elemType = static_cast( type ).elemType(); return cspToProtoCppType( *elemType ); @@ -99,169 +104,230 @@ bool ProtobufHelper::isCoercible( proto::FieldDescriptor::CppType src, CspType:: { switch( src ) { - case proto::FieldDescriptor::CPPTYPE_INT32: return ( dest == CspType::Type::UINT32 || dest == CspType::Type::INT64 || dest == CspType::Type::UINT64 ); - case proto::FieldDescriptor::CPPTYPE_UINT32: return ( dest == CspType::Type::INT32 || dest == CspType::Type::INT64 || dest == CspType::Type::UINT64 ); - case proto::FieldDescriptor::CPPTYPE_INT64: return dest == CspType::Type::UINT64; - case proto::FieldDescriptor::CPPTYPE_UINT64: return dest == CspType::Type::INT64; - case proto::FieldDescriptor::CPPTYPE_FLOAT: return dest == CspType::Type::DOUBLE; - case proto::FieldDescriptor::CPPTYPE_ENUM: return dest == CspType::Type::STRING; + case proto::FieldDescriptor::CPPTYPE_INT32: + return ( dest == CspType::Type::UINT32 || dest == CspType::Type::INT64 || dest == CspType::Type::UINT64 ); + case proto::FieldDescriptor::CPPTYPE_UINT32: + return ( dest == CspType::Type::INT32 || dest == CspType::Type::INT64 || dest == CspType::Type::UINT64 ); + case proto::FieldDescriptor::CPPTYPE_INT64: + return dest == CspType::Type::UINT64; + case proto::FieldDescriptor::CPPTYPE_UINT64: + return dest == CspType::Type::INT64; + case proto::FieldDescriptor::CPPTYPE_FLOAT: + return dest == CspType::Type::DOUBLE; + case proto::FieldDescriptor::CPPTYPE_ENUM: + return dest == CspType::Type::STRING; default: return false; } } template<> -int32_t ProtobufHelper::coercedValue( const proto::Reflection * access, const proto::Message & message, const proto::FieldDescriptor * descr, int index ) +int32_t ProtobufHelper::coercedValue( const proto::Reflection * access, const proto::Message & message, + const proto::FieldDescriptor * descr, int index ) { - switch( descr -> cpp_type() ) + switch( descr->cpp_type() ) { - case proto::FieldDescriptor::CPPTYPE_INT32: return index == -1 ? access -> GetInt32( message, descr ) : access -> GetRepeatedInt32( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_INT32: + return index == -1 ? access->GetInt32( message, descr ) : access->GetRepeatedInt32( message, descr, index ); case proto::FieldDescriptor::CPPTYPE_UINT32: { - uint32_t v = index == -1 ? access -> GetUInt32( message, descr ) : access -> GetRepeatedUInt32( message, descr, index ); - if( v > uint32_t(std::numeric_limits::max()) ) - CSP_THROW( RangeError, "coercion out of range for UINT32 value into INT32 value for proto msg type " << message.GetTypeName() << " field " << descr -> name() ); - return ( int32_t ) v; + uint32_t v = index == -1 ? access->GetUInt32( message, descr ) + : access->GetRepeatedUInt32( message, descr, index ); + if( v > uint32_t( std::numeric_limits::max() ) ) + CSP_THROW( RangeError, + "coercion out of range for UINT32 value into INT32 value for proto msg type " + << message.GetTypeName() << " field " << descr->name() ); + return (int32_t)v; } default: - CSP_THROW( TypeError, "Attempting to coerce proto field type " << descr -> cpp_type_name() << " to int32_t" ); + CSP_THROW( TypeError, "Attempting to coerce proto field type " << descr->cpp_type_name() << " to int32_t" ); } } template<> -uint32_t ProtobufHelper::coercedValue( const proto::Reflection * access, const proto::Message & message, const proto::FieldDescriptor * descr, int index ) +uint32_t ProtobufHelper::coercedValue( const proto::Reflection * access, const proto::Message & message, + const proto::FieldDescriptor * descr, int index ) { - switch( descr -> cpp_type() ) + switch( descr->cpp_type() ) { - case proto::FieldDescriptor::CPPTYPE_UINT32: return index == -1 ? access -> GetUInt32( message, descr ) : access -> GetRepeatedUInt32( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_UINT32: + return index == -1 ? access->GetUInt32( message, descr ) + : access->GetRepeatedUInt32( message, descr, index ); case proto::FieldDescriptor::CPPTYPE_INT32: { - int32_t v = index == -1 ? access -> GetInt32( message, descr ) : access -> GetRepeatedInt32( message, descr, index ); + int32_t v + = index == -1 ? access->GetInt32( message, descr ) : access->GetRepeatedInt32( message, descr, index ); if( v < 0 ) - CSP_THROW( RangeError, "coercion out of range for INT32 value into uint32_t value for proto msg type " << message.GetTypeName() << " field " << descr -> name() ); - return ( uint32_t ) v; + CSP_THROW( RangeError, + "coercion out of range for INT32 value into uint32_t value for proto msg type " + << message.GetTypeName() << " field " << descr->name() ); + return (uint32_t)v; } default: - CSP_THROW( TypeError, "Attempting to coerce proto field type " << descr -> cpp_type_name() << " to uint32_t" ); + CSP_THROW( TypeError, + "Attempting to coerce proto field type " << descr->cpp_type_name() << " to uint32_t" ); } } template<> -int64_t ProtobufHelper::coercedValue( const proto::Reflection * access, const proto::Message & message, const proto::FieldDescriptor * descr, int index ) +int64_t ProtobufHelper::coercedValue( const proto::Reflection * access, const proto::Message & message, + const proto::FieldDescriptor * descr, int index ) { - switch( descr -> cpp_type() ) + switch( descr->cpp_type() ) { - case proto::FieldDescriptor::CPPTYPE_INT64: return index == -1 ? access -> GetInt64( message, descr ) : access -> GetRepeatedInt64( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_INT64: + return index == -1 ? access->GetInt64( message, descr ) : access->GetRepeatedInt64( message, descr, index ); case proto::FieldDescriptor::CPPTYPE_UINT64: { - uint64_t v = index == -1 ? access -> GetUInt64( message, descr ) : access -> GetRepeatedUInt64( message, descr, index ); - if( v > uint64_t(std::numeric_limits::max()) ) - CSP_THROW( RangeError, "coercion out of range for UINT64 value into int64_t value for proto msg type " << message.GetTypeName() << " field " << descr -> name() ); - return ( int64_t ) v; + uint64_t v = index == -1 ? access->GetUInt64( message, descr ) + : access->GetRepeatedUInt64( message, descr, index ); + if( v > uint64_t( std::numeric_limits::max() ) ) + CSP_THROW( RangeError, + "coercion out of range for UINT64 value into int64_t value for proto msg type " + << message.GetTypeName() << " field " << descr->name() ); + return (int64_t)v; } - case proto::FieldDescriptor::CPPTYPE_UINT32: return index == -1 ? access -> GetUInt32( message, descr ) : access -> GetRepeatedUInt32( message, descr, index ); - case proto::FieldDescriptor::CPPTYPE_INT32: return index == -1 ? access -> GetInt32( message, descr ) : access -> GetRepeatedInt32( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_UINT32: + return index == -1 ? access->GetUInt32( message, descr ) + : access->GetRepeatedUInt32( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_INT32: + return index == -1 ? access->GetInt32( message, descr ) : access->GetRepeatedInt32( message, descr, index ); default: - CSP_THROW( TypeError, "Attempting to coerce proto field type " << descr -> cpp_type_name() << " to int64_t" ); + CSP_THROW( TypeError, "Attempting to coerce proto field type " << descr->cpp_type_name() << " to int64_t" ); } } template<> -uint64_t ProtobufHelper::coercedValue( const proto::Reflection * access, const proto::Message & message, const proto::FieldDescriptor * descr, int index ) +uint64_t ProtobufHelper::coercedValue( const proto::Reflection * access, const proto::Message & message, + const proto::FieldDescriptor * descr, int index ) { - switch( descr -> cpp_type() ) + switch( descr->cpp_type() ) { - case proto::FieldDescriptor::CPPTYPE_UINT64: return index == -1 ? access -> GetUInt64( message, descr ) : access -> GetRepeatedUInt64( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_UINT64: + return index == -1 ? access->GetUInt64( message, descr ) + : access->GetRepeatedUInt64( message, descr, index ); case proto::FieldDescriptor::CPPTYPE_INT64: { - int64_t v = index == -1 ? access -> GetInt64( message, descr ) : access -> GetRepeatedInt64( message, descr, index ); + int64_t v + = index == -1 ? access->GetInt64( message, descr ) : access->GetRepeatedInt64( message, descr, index ); if( v < 0 ) - CSP_THROW( RangeError, "coercion out of range for INT64 value into uint64_t value for proto msg type " << message.GetTypeName() << " field " << descr -> name() ); - return ( uint64_t ) v; + CSP_THROW( RangeError, + "coercion out of range for INT64 value into uint64_t value for proto msg type " + << message.GetTypeName() << " field " << descr->name() ); + return (uint64_t)v; } - case proto::FieldDescriptor::CPPTYPE_UINT32: return index == -1 ? access -> GetUInt32( message, descr ) : access -> GetRepeatedUInt32( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_UINT32: + return index == -1 ? access->GetUInt32( message, descr ) + : access->GetRepeatedUInt32( message, descr, index ); case proto::FieldDescriptor::CPPTYPE_INT32: { - int32_t v = index == -1 ? access -> GetInt32( message, descr ) : access -> GetRepeatedInt32( message, descr, index ); + int32_t v + = index == -1 ? access->GetInt32( message, descr ) : access->GetRepeatedInt32( message, descr, index ); if( v < 0 ) - CSP_THROW( RangeError, "coercion out of range for INT32 value into uint64_t value for proto msg type " << message.GetTypeName() << " field " << descr -> name() ); - return ( uint64_t ) v; + CSP_THROW( RangeError, + "coercion out of range for INT32 value into uint64_t value for proto msg type " + << message.GetTypeName() << " field " << descr->name() ); + return (uint64_t)v; } default: - CSP_THROW( TypeError, "Attempting to coerce proto field type " << descr -> cpp_type_name() << " to int64_t" ); + CSP_THROW( TypeError, "Attempting to coerce proto field type " << descr->cpp_type_name() << " to int64_t" ); } } template<> -double ProtobufHelper::coercedValue( const proto::Reflection * access, const proto::Message & message, const proto::FieldDescriptor * descr, int index ) +double ProtobufHelper::coercedValue( const proto::Reflection * access, const proto::Message & message, + const proto::FieldDescriptor * descr, int index ) { - switch( descr -> cpp_type() ) + switch( descr->cpp_type() ) { - case proto::FieldDescriptor::CPPTYPE_INT32: return index == -1 ? access -> GetInt32( message, descr ) : access -> GetRepeatedInt32( message, descr, index ); - case proto::FieldDescriptor::CPPTYPE_UINT32: return index == -1 ? access -> GetUInt32( message, descr ) : access -> GetRepeatedUInt32( message, descr, index ); - case proto::FieldDescriptor::CPPTYPE_INT64: return index == -1 ? access -> GetInt64( message, descr ) : access -> GetRepeatedInt64( message, descr, index ); - case proto::FieldDescriptor::CPPTYPE_UINT64: return index == -1 ? access -> GetUInt64( message, descr ) : access -> GetRepeatedUInt64( message, descr, index ); - case proto::FieldDescriptor::CPPTYPE_FLOAT: return index == -1 ? access -> GetFloat( message, descr ) : access -> GetRepeatedFloat( message, descr, index ); - case proto::FieldDescriptor::CPPTYPE_DOUBLE: return index == -1 ? access -> GetDouble( message, descr ) : access -> GetRepeatedDouble( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_INT32: + return index == -1 ? access->GetInt32( message, descr ) : access->GetRepeatedInt32( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_UINT32: + return index == -1 ? access->GetUInt32( message, descr ) + : access->GetRepeatedUInt32( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_INT64: + return index == -1 ? access->GetInt64( message, descr ) : access->GetRepeatedInt64( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_UINT64: + return index == -1 ? access->GetUInt64( message, descr ) + : access->GetRepeatedUInt64( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_FLOAT: + return index == -1 ? access->GetFloat( message, descr ) : access->GetRepeatedFloat( message, descr, index ); + case proto::FieldDescriptor::CPPTYPE_DOUBLE: + return index == -1 ? access->GetDouble( message, descr ) + : access->GetRepeatedDouble( message, descr, index ); default: - CSP_THROW( TypeError, "Attempting to coerce proto field type " << descr -> cpp_type_name() << " to double" ); + CSP_THROW( TypeError, "Attempting to coerce proto field type " << descr->cpp_type_name() << " to double" ); } } -void ProtobufStructMapper::init( const CspTypePtr & type, const Dictionary & fieldMap, const google::protobuf::Descriptor * protoDesc ) +void ProtobufStructMapper::init( const CspTypePtr & type, const Dictionary & fieldMap, + const google::protobuf::Descriptor * protoDesc ) { m_fields = buildFields( type, fieldMap, protoDesc ); } -ProtobufStructMapper::Fields ProtobufStructMapper::buildFields( const CspTypePtr & type, const Dictionary & fieldMap, const google::protobuf::Descriptor * protoDesc ) +ProtobufStructMapper::Fields ProtobufStructMapper::buildFields( const CspTypePtr & type, const Dictionary & fieldMap, + const google::protobuf::Descriptor * protoDesc ) { auto structType = std::static_pointer_cast( type ); - auto structMeta = structType -> meta(); + auto structMeta = structType->meta(); Fields fields; for( auto it = fieldMap.begin(); it != fieldMap.end(); ++it ) { - std::string protoField = it.key(); + std::string protoField = it.key(); std::string structField; DictionaryPtr subFieldMap; if( it.hasValue() ) structField = it.value(); else { - //for sub-structs we expect value to by a dict of { "structfield" : fieldMap } + // for sub-structs we expect value to by a dict of { "structfield" : fieldMap } auto subdict = it.value(); - if( subdict -> size() != 1 ) - CSP_THROW( ValueError, "Expected sub-fieldmap for incoming protofield " << protoField << " to have a single key : map entry" ); + if( subdict->size() != 1 ) + CSP_THROW( ValueError, + "Expected sub-fieldmap for incoming protofield " << protoField + << " to have a single key : map entry" ); - structField = subdict -> begin().key(); - subFieldMap = subdict -> begin().value(); + structField = subdict->begin().key(); + subFieldMap = subdict->begin().value(); } - auto sField = structMeta -> field( structField ); + auto sField = structMeta->field( structField ); if( !sField ) - CSP_THROW( ValueError, "field " << structField << " is not a valid field on struct type " << structMeta -> name() ); + CSP_THROW( ValueError, + "field " << structField << " is not a valid field on struct type " << structMeta->name() ); - const proto::FieldDescriptor * protoFieldDesc = protoDesc -> FindFieldByName( protoField ); + const proto::FieldDescriptor * protoFieldDesc = protoDesc->FindFieldByName( protoField ); if( !protoFieldDesc ) - CSP_THROW( ValueError, "field '" << protoField << "' is not a valid field on proto descriptor for " << protoDesc -> name() ); - - auto protoType = ProtobufHelper::cspToProtoCppType( *sField -> type() ); - if( ( protoType != protoFieldDesc -> cpp_type() && !ProtobufHelper::isCoercible( protoFieldDesc -> cpp_type(), sField -> type() -> type() ) ) || - ( sField -> type() -> type() == CspType::Type::ARRAY && !protoFieldDesc -> is_repeated() ) ) + CSP_THROW( ValueError, + "field '" << protoField << "' is not a valid field on proto descriptor for " + << protoDesc->name() ); + + auto protoType = ProtobufHelper::cspToProtoCppType( *sField->type() ); + if( ( protoType != protoFieldDesc->cpp_type() + && !ProtobufHelper::isCoercible( protoFieldDesc->cpp_type(), sField->type()->type() ) ) + || ( sField->type()->type() == CspType::Type::ARRAY && !protoFieldDesc->is_repeated() ) ) { - CSP_THROW( TypeError, "Type mismatch on struct field " << structField << " with type " << sField -> type() -> type() << " and protofield " << - protoField << " with type " << ( protoFieldDesc -> is_repeated() ? "repeated " : "" ) << protoFieldDesc -> cpp_type_name() ); + CSP_THROW( TypeError, + "Type mismatch on struct field " << structField << " with type " << sField->type()->type() + << " and protofield " << protoField << " with type " + << ( protoFieldDesc->is_repeated() ? "repeated " : "" ) + << protoFieldDesc->cpp_type_name() ); } std::shared_ptr subfields; - if( sField -> type() -> type() == CspType::Type::STRUCT ) + if( sField->type()->type() == CspType::Type::STRUCT ) { if( !subFieldMap ) - CSP_THROW( ValueError, "invalid field_map entry for sub-struct field " << sField -> fieldname() << " on struct type " << structMeta -> name() ); - subfields = std::make_shared( buildFields( sField -> type(), *subFieldMap, protoFieldDesc -> message_type() ) ); + CSP_THROW( ValueError, + "invalid field_map entry for sub-struct field " << sField->fieldname() << " on struct type " + << structMeta->name() ); + subfields = std::make_shared( + buildFields( sField->type(), *subFieldMap, protoFieldDesc->message_type() ) ); } fields.emplace_back( FieldEntry{ protoFieldDesc, sField.get(), subfields } ); } @@ -269,14 +335,10 @@ ProtobufStructMapper::Fields ProtobufStructMapper::buildFields( const CspTypePtr return fields; } -using SupportedArrayCspTypeSwitch = PartialSwitchCspType; +using SupportedArrayCspTypeSwitch + = PartialSwitchCspType; template static T extractRepeatedValue( const proto::Message & protoMsg, const proto::FieldDescriptor * field, int index ); @@ -284,7 +346,7 @@ static T extractRepeatedValue( const proto::Message & protoMsg, const proto::Fie template<> bool extractRepeatedValue( const proto::Message & protoMsg, const proto::FieldDescriptor * field, int index ) { - return protoMsg.GetReflection() -> GetRepeatedBool( protoMsg, field, index ); + return protoMsg.GetReflection()->GetRepeatedBool( protoMsg, field, index ); } template<> @@ -320,10 +382,11 @@ double extractRepeatedValue( const proto::Message & protoMsg, const proto::Field template<> std::string extractRepeatedValue( const proto::Message & protoMsg, const proto::FieldDescriptor * field, int index ) { - return protoMsg.GetReflection() -> GetRepeatedString( protoMsg, field, index ); + return protoMsg.GetReflection()->GetRepeatedString( protoMsg, field, index ); } -void ProtobufStructMapper::mapProtoToStruct( StructPtr & struct_, const proto::Message & protoMsg, const Fields & fields ) +void ProtobufStructMapper::mapProtoToStruct( StructPtr & struct_, const proto::Message & protoMsg, + const Fields & fields ) { const proto::Reflection * protoAccess = protoMsg.GetReflection(); @@ -332,58 +395,78 @@ void ProtobufStructMapper::mapProtoToStruct( StructPtr & struct_, const proto::M auto * pField = entry.pField; auto * sField = entry.sField; - switch( sField -> type() -> type() ) + switch( sField->type()->type() ) { - case CspType::Type::BOOL: sField -> setValue( struct_.get(), protoAccess -> GetBool( protoMsg, pField ) ); break; - case CspType::Type::INT32: sField -> setValue( struct_.get(), ProtobufHelper::coercedValue( protoAccess, protoMsg, pField ) ); break; - case CspType::Type::UINT32: sField -> setValue( struct_.get(), ProtobufHelper::coercedValue( protoAccess, protoMsg, pField ) ); break; - case CspType::Type::INT64: sField -> setValue( struct_.get(), ProtobufHelper::coercedValue( protoAccess, protoMsg, pField ) ); break; - case CspType::Type::UINT64: sField -> setValue( struct_.get(), ProtobufHelper::coercedValue( protoAccess, protoMsg, pField ) ); break; - case CspType::Type::DOUBLE: sField -> setValue( struct_.get(), ProtobufHelper::coercedValue( protoAccess, protoMsg, pField ) ); break; + case CspType::Type::BOOL: + sField->setValue( struct_.get(), protoAccess->GetBool( protoMsg, pField ) ); + break; + case CspType::Type::INT32: + sField->setValue( struct_.get(), + ProtobufHelper::coercedValue( protoAccess, protoMsg, pField ) ); + break; + case CspType::Type::UINT32: + sField->setValue( struct_.get(), + ProtobufHelper::coercedValue( protoAccess, protoMsg, pField ) ); + break; + case CspType::Type::INT64: + sField->setValue( struct_.get(), + ProtobufHelper::coercedValue( protoAccess, protoMsg, pField ) ); + break; + case CspType::Type::UINT64: + sField->setValue( struct_.get(), + ProtobufHelper::coercedValue( protoAccess, protoMsg, pField ) ); + break; + case CspType::Type::DOUBLE: + sField->setValue( struct_.get(), + ProtobufHelper::coercedValue( protoAccess, protoMsg, pField ) ); + break; case CspType::Type::STRING: { - if( pField -> cpp_type() == proto::FieldDescriptor::CPPTYPE_ENUM ) + if( pField->cpp_type() == proto::FieldDescriptor::CPPTYPE_ENUM ) { - sField -> setValue( struct_.get(), protoAccess -> GetEnum( protoMsg, pField ) -> name() ); + sField->setValue( struct_.get(), protoAccess->GetEnum( protoMsg, pField )->name() ); break; } else { - sField -> setValue( struct_.get(), protoAccess -> GetString( protoMsg, pField ) ); + sField->setValue( struct_.get(), protoAccess->GetString( protoMsg, pField ) ); break; } } case CspType::Type::STRUCT: { - auto subStructMeta = static_cast( sField ) -> meta(); - auto subStruct = subStructMeta -> create(); - mapProtoToStruct( subStruct, protoAccess -> GetMessage( protoMsg, pField ), *entry.subFields ); - sField -> setValue( struct_.get(), subStruct ); + auto subStructMeta = static_cast( sField )->meta(); + auto subStruct = subStructMeta->create(); + mapProtoToStruct( subStruct, protoAccess->GetMessage( protoMsg, pField ), *entry.subFields ); + sField->setValue( struct_.get(), subStruct ); break; } case CspType::Type::ARRAY: { - int count = protoAccess -> FieldSize( protoMsg, pField ); - - auto elemType = static_cast( *sField -> type() ).elemType(); - SupportedArrayCspTypeSwitch::invoke( elemType.get(), [&count,&protoMsg,&pField,&sField,&struct_]( auto tag ) - { - using ElemT = typename decltype(tag)::type; - using StorageT = typename CspType::Type::toCArrayStorageType::type; - std::vector data; - data.reserve( count ); - for( int i = 0; i < count; ++i ) - data.emplace_back( extractRepeatedValue( protoMsg, pField, i ) ); - sField -> setValue>( struct_.get(), std::move( data ) ); - } ); + int count = protoAccess->FieldSize( protoMsg, pField ); + + auto elemType = static_cast( *sField->type() ).elemType(); + SupportedArrayCspTypeSwitch::invoke( + elemType.get(), + [&count, &protoMsg, &pField, &sField, &struct_]( auto tag ) + { + using ElemT = typename decltype( tag )::type; + using StorageT = typename CspType::Type::toCArrayStorageType::type; + std::vector data; + data.reserve( count ); + for( int i = 0; i < count; ++i ) + data.emplace_back( extractRepeatedValue( protoMsg, pField, i ) ); + sField->setValue>( struct_.get(), std::move( data ) ); + } ); break; } default: - CSP_THROW( TypeError, "Struct field type " << sField -> type() -> type() << " not currently mappable to proto field" ); + CSP_THROW( TypeError, + "Struct field type " << sField->type()->type() << " not currently mappable to proto field" ); } } } -} +} // namespace csp::adapters::utils diff --git a/cpp/csp/adapters/utils/ProtobufMessageStructConverter.cpp b/cpp/csp/adapters/utils/ProtobufMessageStructConverter.cpp index daf8d4bf..5e6803e6 100644 --- a/cpp/csp/adapters/utils/ProtobufMessageStructConverter.cpp +++ b/cpp/csp/adapters/utils/ProtobufMessageStructConverter.cpp @@ -1,6 +1,6 @@ #include -#include #include +#include #include namespace proto = google::protobuf; @@ -8,19 +8,20 @@ namespace proto = google::protobuf; namespace csp::adapters::utils { -ProtobufMessageStructConverter::ProtobufMessageStructConverter( const CspTypePtr & type, const Dictionary & properties ) : MessageStructConverter( type, properties ) +ProtobufMessageStructConverter::ProtobufMessageStructConverter( const CspTypePtr & type, const Dictionary & properties ) + : MessageStructConverter( type, properties ) { - if( type -> type() != CspType::Type::STRUCT ) - CSP_THROW( TypeError, "ProtobufMessageStructConverter expects type struct got " << type -> type() ); + if( type->type() != CspType::Type::STRUCT ) + CSP_THROW( TypeError, "ProtobufMessageStructConverter expects type struct got " << type->type() ); - const std::string &protodir = properties.get( "proto_directory" ); - const std::string &protofile = properties.get( "proto_filename" ); - const std::string &protomsg = properties.get( "proto_message" ); - const Dictionary & fieldMap = *properties.get( "field_map" ); + const std::string & protodir = properties.get( "proto_directory" ); + const std::string & protofile = properties.get( "proto_filename" ); + const std::string & protomsg = properties.get( "proto_message" ); + const Dictionary & fieldMap = *properties.get( "field_map" ); const proto::FileDescriptor * protoFileDesc = ProtobufHelper::instance().import( protodir, protofile ); - m_protoDesc = protoFileDesc -> FindMessageTypeByName( protomsg ); + m_protoDesc = protoFileDesc->FindMessageTypeByName( protomsg ); if( !m_protoDesc ) CSP_THROW( ValueError, "Failed to find proto message " << protomsg << " in proto schema " << protofile ); @@ -29,18 +30,19 @@ ProtobufMessageStructConverter::ProtobufMessageStructConverter( const CspTypePtr csp::StructPtr ProtobufMessageStructConverter::asStruct( void * bytes, size_t size ) { - auto struct_ = m_structMeta -> create(); + auto struct_ = m_structMeta->create(); auto * prototype = ProtobufHelper::instance().getMessage( m_protoDesc ); if( !prototype ) - CSP_THROW( RuntimeException, "Failed to get proto message instance for proto descriptor " << m_protoDesc -> name() ); + CSP_THROW( RuntimeException, + "Failed to get proto message instance for proto descriptor " << m_protoDesc->name() ); - std::unique_ptr protoMsg( prototype -> New() ); - if( !protoMsg -> ParseFromArray( bytes, size ) ) - CSP_THROW( RuntimeException, "Failed to parse proto message on " << m_protoDesc -> name() ); + std::unique_ptr protoMsg( prototype->New() ); + if( !protoMsg->ParseFromArray( bytes, size ) ) + CSP_THROW( RuntimeException, "Failed to parse proto message on " << m_protoDesc->name() ); m_protoMapper.mapProtoToStruct( struct_, *protoMsg ); return struct_; } -} +} // namespace csp::adapters::utils diff --git a/cpp/csp/adapters/utils/RawBytesMessageStructConverter.cpp b/cpp/csp/adapters/utils/RawBytesMessageStructConverter.cpp index 4eaa87e3..89a7dbb9 100644 --- a/cpp/csp/adapters/utils/RawBytesMessageStructConverter.cpp +++ b/cpp/csp/adapters/utils/RawBytesMessageStructConverter.cpp @@ -3,9 +3,9 @@ namespace csp::adapters::utils { -RawBytesMessageStructConverter::RawBytesMessageStructConverter( const CspTypePtr & type, - const Dictionary & properties ) : MessageStructConverter( type, properties ), - m_targetField( nullptr ) +RawBytesMessageStructConverter::RawBytesMessageStructConverter( const CspTypePtr & type, const Dictionary & properties ) + : MessageStructConverter( type, properties ) + , m_targetField( nullptr ) { const Dictionary & fieldMap = *properties.get( "field_map" ); if( fieldMap.size() > 1 ) @@ -13,31 +13,35 @@ RawBytesMessageStructConverter::RawBytesMessageStructConverter( const CspTypePtr if( fieldMap.size() == 1 ) { - if( type -> type() != CspType::Type::STRUCT ) - CSP_THROW( ValueError, "field_map provided on non-struct type " << type -> type() << " in adapter" ); + if( type->type() != CspType::Type::STRUCT ) + CSP_THROW( ValueError, "field_map provided on non-struct type " << type->type() << " in adapter" ); if( !fieldMap.exists( "" ) ) - CSP_THROW( ValueError, "RawBytesMessageStructConverter expects one entry in fieldMap with empty string as source key" ); + CSP_THROW( ValueError, + "RawBytesMessageStructConverter expects one entry in fieldMap with empty string as source key" ); auto targetKey = fieldMap.get( "" ); - auto sField = m_structMeta -> field( targetKey ); - if( !sField || sField -> type() -> type() != CspType::Type::STRING ) - CSP_THROW( TypeError, "field " << targetKey << " on struct " << m_structMeta -> name() << ( sField ? "is not string type" : "does not exist" ) ); + auto sField = m_structMeta->field( targetKey ); + if( !sField || sField->type()->type() != CspType::Type::STRING ) + CSP_THROW( TypeError, + "field " << targetKey << " on struct " << m_structMeta->name() + << ( sField ? "is not string type" : "does not exist" ) ); m_targetField = static_cast( sField.get() ); } - else if( type -> type() != CspType::Type::STRING ) - CSP_THROW( TypeError, "TestMessageStructConverter expected type of STRING for empty field_map got " << type -> type() ); + else if( type->type() != CspType::Type::STRING ) + CSP_THROW( TypeError, + "TestMessageStructConverter expected type of STRING for empty field_map got " << type->type() ); } csp::StructPtr RawBytesMessageStructConverter::asStruct( void * bytes, size_t size ) { - if( m_type -> type() == CspType::Type::STRUCT ) + if( m_type->type() == CspType::Type::STRUCT ) { - StructPtr data = m_structMeta -> create(); - m_targetField -> setValue( data.get(), std::string( ( const char * ) bytes, size ) ); + StructPtr data = m_structMeta->create(); + m_targetField->setValue( data.get(), std::string( (const char *)bytes, size ) ); return data; } else - abort(); //TBD doesnt fit asStruct API + abort(); // TBD doesnt fit asStruct API } -} +} // namespace csp::adapters::utils diff --git a/cpp/csp/adapters/websocket/ClientAdapterManager.cpp b/cpp/csp/adapters/websocket/ClientAdapterManager.cpp index 423f2a23..d776ce31 100644 --- a/cpp/csp/adapters/websocket/ClientAdapterManager.cpp +++ b/cpp/csp/adapters/websocket/ClientAdapterManager.cpp @@ -1,122 +1,126 @@ #include -namespace csp { +namespace csp +{ -INIT_CSP_ENUM( adapters::websocket::ClientStatusType, - "ACTIVE", - "GENERIC_ERROR", - "CONNECTION_FAILED", - "CLOSED", - "MESSAGE_SEND_FAIL", -); +INIT_CSP_ENUM( adapters::websocket::ClientStatusType, "ACTIVE", "GENERIC_ERROR", "CONNECTION_FAILED", "CLOSED", + "MESSAGE_SEND_FAIL", ); } // With TLS -namespace csp::adapters::websocket { - -ClientAdapterManager::ClientAdapterManager( Engine* engine, const Dictionary & properties ) -: AdapterManager( engine ), - m_active( false ), - m_shouldRun( false ), - m_endpoint( std::make_unique( properties ) ), - m_inputAdapter( nullptr ), - m_outputAdapter( nullptr ), - m_updateAdapter( nullptr ), - m_thread( nullptr ), - m_properties( properties ) -{ }; - -ClientAdapterManager::~ClientAdapterManager() -{ }; +namespace csp::adapters::websocket +{ + +ClientAdapterManager::ClientAdapterManager( Engine * engine, const Dictionary & properties ) + : AdapterManager( engine ) + , m_active( false ) + , m_shouldRun( false ) + , m_endpoint( std::make_unique( properties ) ) + , m_inputAdapter( nullptr ) + , m_outputAdapter( nullptr ) + , m_updateAdapter( nullptr ) + , m_thread( nullptr ) + , m_properties( properties ) {}; + +ClientAdapterManager::~ClientAdapterManager() {}; void ClientAdapterManager::start( DateTime starttime, DateTime endtime ) { AdapterManager::start( starttime, endtime ); m_shouldRun = true; - m_endpoint -> setOnOpen( - [ this ]() { + m_endpoint->setOnOpen( + [this]() + { m_active = true; pushStatus( StatusLevel::INFO, ClientStatusType::ACTIVE, "Connected successfully" ); - } - ); - m_endpoint -> setOnFail( - [ this ]( const std::string& reason ) { + } ); + m_endpoint->setOnFail( + [this]( const std::string & reason ) + { std::stringstream ss; ss << "Connection Failure: " << reason; m_active = false; pushStatus( StatusLevel::ERROR, ClientStatusType::CONNECTION_FAILED, ss.str() ); - } - ); - if( m_inputAdapter ) { - m_endpoint -> setOnMessage( - [ this ]( void* c, size_t t ) { - PushBatch batch( m_engine -> rootEngine() ); - m_inputAdapter -> processMessage( c, t, &batch ); - } - ); - } else { + } ); + if( m_inputAdapter ) + { + m_endpoint->setOnMessage( + [this]( void * c, size_t t ) + { + PushBatch batch( m_engine->rootEngine() ); + m_inputAdapter->processMessage( c, t, &batch ); + } ); + } + else + { // if a user doesn't call WebsocketAdapterManager.subscribe, no inputadapter will be created // but we still need something to avoid on_message_cb not being set in the endpoint. - m_endpoint -> setOnMessage( []( void* c, size_t t ){} ); + m_endpoint->setOnMessage( []( void * c, size_t t ) {} ); } - m_endpoint -> setOnClose( - [ this ]() { + m_endpoint->setOnClose( + [this]() + { m_active = false; pushStatus( StatusLevel::INFO, ClientStatusType::CLOSED, "Connection closed" ); - } - ); - m_endpoint -> setOnSendFail( - [ this ]( const std::string& s ) { + } ); + m_endpoint->setOnSendFail( + [this]( const std::string & s ) + { std::stringstream ss; ss << "Failed to send: " << s; pushStatus( StatusLevel::ERROR, ClientStatusType::MESSAGE_SEND_FAIL, ss.str() ); - } - ); + } ); - m_thread = std::make_unique( [ this ]() { - while( m_shouldRun ) + m_thread = std::make_unique( + [this]() { - m_endpoint -> run(); - m_active = false; - if( m_shouldRun ) sleep( m_properties.get( "reconnect_interval" ) ); - } - }); + while( m_shouldRun ) + { + m_endpoint->run(); + m_active = false; + if( m_shouldRun ) + sleep( m_properties.get( "reconnect_interval" ) ); + } + } ); }; -void ClientAdapterManager::stop() { +void ClientAdapterManager::stop() +{ AdapterManager::stop(); - m_shouldRun=false; - if( m_active ) m_endpoint->stop(); - if( m_thread ) m_thread->join(); + m_shouldRun = false; + if( m_active ) + m_endpoint->stop(); + if( m_thread ) + m_thread->join(); }; -PushInputAdapter* ClientAdapterManager::getInputAdapter(CspTypePtr & type, PushMode pushMode, const Dictionary & properties) +PushInputAdapter * ClientAdapterManager::getInputAdapter( CspTypePtr & type, PushMode pushMode, + const Dictionary & properties ) { - if (m_inputAdapter == nullptr) + if( m_inputAdapter == nullptr ) { - m_inputAdapter = m_engine -> createOwnedObject( + m_inputAdapter = m_engine->createOwnedObject( // m_engine, - type, - pushMode, - properties - ); + type, pushMode, properties ); } return m_inputAdapter; }; -OutputAdapter* ClientAdapterManager::getOutputAdapter() +OutputAdapter * ClientAdapterManager::getOutputAdapter() { - if (m_outputAdapter == nullptr) m_outputAdapter = m_engine -> createOwnedObject(*m_endpoint); + if( m_outputAdapter == nullptr ) + m_outputAdapter = m_engine->createOwnedObject( *m_endpoint ); return m_outputAdapter; } OutputAdapter * ClientAdapterManager::getHeaderUpdateAdapter() { - if (m_updateAdapter == nullptr) m_updateAdapter = m_engine -> createOwnedObject( m_endpoint -> getProperties() ); + if( m_updateAdapter == nullptr ) + m_updateAdapter = m_engine->createOwnedObject( m_endpoint->getProperties() ); return m_updateAdapter; } @@ -127,4 +131,4 @@ DateTime ClientAdapterManager::processNextSimTimeSlice( DateTime time ) return DateTime::NONE(); } -} +} // namespace csp::adapters::websocket diff --git a/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.cpp b/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.cpp index 995bd731..e950e680 100644 --- a/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.cpp +++ b/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.cpp @@ -1,20 +1,20 @@ #include -namespace csp::adapters::websocket { +namespace csp::adapters::websocket +{ -ClientHeaderUpdateOutputAdapter::ClientHeaderUpdateOutputAdapter( - Engine * engine, - Dictionary& properties -) : OutputAdapter( engine ), m_properties( properties ) -{ }; +ClientHeaderUpdateOutputAdapter::ClientHeaderUpdateOutputAdapter( Engine * engine, Dictionary & properties ) + : OutputAdapter( engine ) + , m_properties( properties ) {}; void ClientHeaderUpdateOutputAdapter::executeImpl() { - DictionaryPtr headers = m_properties.get("headers"); - for( auto& update : input() -> lastValueTyped>() ) - { - if( update -> key_isSet() && update -> value_isSet() ) headers->update( update->key(), update->value() ); + DictionaryPtr headers = m_properties.get( "headers" ); + for( auto & update : input()->lastValueTyped>() ) + { + if( update->key_isSet() && update->value_isSet() ) + headers->update( update->key(), update->value() ); } }; -} \ No newline at end of file +} // namespace csp::adapters::websocket \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/ClientInputAdapter.cpp b/cpp/csp/adapters/websocket/ClientInputAdapter.cpp index 548b0bc0..8abcdb64 100644 --- a/cpp/csp/adapters/websocket/ClientInputAdapter.cpp +++ b/cpp/csp/adapters/websocket/ClientInputAdapter.cpp @@ -3,40 +3,36 @@ namespace csp::adapters::websocket { -ClientInputAdapter::ClientInputAdapter( - Engine * engine, - CspTypePtr & type, - PushMode pushMode, - const Dictionary & properties -) : PushInputAdapter(engine, type, pushMode) +ClientInputAdapter::ClientInputAdapter( Engine * engine, CspTypePtr & type, PushMode pushMode, + const Dictionary & properties ) + : PushInputAdapter( engine, type, pushMode ) { - if( type -> type() != CspType::Type::STRUCT && - type -> type() != CspType::Type::STRING ) - CSP_THROW( RuntimeException, "Unsupported type: " << type -> type() ); + if( type->type() != CspType::Type::STRUCT && type->type() != CspType::Type::STRING ) + CSP_THROW( RuntimeException, "Unsupported type: " << type->type() ); if( properties.exists( "meta_field_map" ) ) { const Dictionary & metaFieldMap = *properties.get( "meta_field_map" ); - if( !metaFieldMap.empty() && type -> type() != CspType::Type::STRUCT ) + if( !metaFieldMap.empty() && type->type() != CspType::Type::STRUCT ) CSP_THROW( ValueError, "meta_field_map is not supported on non-struct types" ); } m_converter = adapters::utils::MessageStructConverterCache::instance().create( type, properties ); }; -void ClientInputAdapter::processMessage( void* c, size_t t, PushBatch* batch ) +void ClientInputAdapter::processMessage( void * c, size_t t, PushBatch * batch ) { - if( type() -> type() == CspType::Type::STRUCT ) + if( type()->type() == CspType::Type::STRUCT ) { - auto tick = m_converter -> asStruct( c, t ); - pushTick( std::move(tick), batch ); - } else if ( type() -> type() == CspType::Type::STRING ) + auto tick = m_converter->asStruct( c, t ); + pushTick( std::move( tick ), batch ); + } + else if( type()->type() == CspType::Type::STRING ) { - pushTick( std::string((char const*)c, t), batch ); + pushTick( std::string( (char const *)c, t ), batch ); } - } -} \ No newline at end of file +} // namespace csp::adapters::websocket \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp b/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp index 3ef3c91a..43881fed 100644 --- a/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp +++ b/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp @@ -1,17 +1,16 @@ #include -namespace csp::adapters::websocket { +namespace csp::adapters::websocket +{ -ClientOutputAdapter::ClientOutputAdapter( - Engine * engine, - WebsocketEndpoint& endpoint -) : OutputAdapter( engine ), m_endpoint( endpoint ) -{ }; +ClientOutputAdapter::ClientOutputAdapter( Engine * engine, WebsocketEndpoint & endpoint ) + : OutputAdapter( engine ) + , m_endpoint( endpoint ) {}; void ClientOutputAdapter::executeImpl() { - const std::string & value = input() -> lastValueTyped(); + const std::string & value = input()->lastValueTyped(); m_endpoint.send( value ); }; -} \ No newline at end of file +} // namespace csp::adapters::websocket \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp b/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp index 8e856c79..6d3c5372 100644 --- a/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp +++ b/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp @@ -1,52 +1,34 @@ #include -namespace csp::adapters::websocket { +namespace csp::adapters::websocket +{ using namespace csp; -WebsocketEndpoint::WebsocketEndpoint( - Dictionary properties -) : m_properties(properties) -{ }; -void WebsocketEndpoint::setOnOpen(void_cb on_open) -{ m_on_open = std::move(on_open); } -void WebsocketEndpoint::setOnFail(string_cb on_fail) -{ m_on_fail = std::move(on_fail); } -void WebsocketEndpoint::setOnMessage(char_cb on_message) -{ m_on_message = std::move(on_message); } -void WebsocketEndpoint::setOnClose(void_cb on_close) -{ m_on_close = std::move(on_close); } -void WebsocketEndpoint::setOnSendFail(string_cb on_send_fail) -{ m_on_send_fail = std::move(on_send_fail); } +WebsocketEndpoint::WebsocketEndpoint( Dictionary properties ) + : m_properties( properties ) {}; +void WebsocketEndpoint::setOnOpen( void_cb on_open ) { m_on_open = std::move( on_open ); } +void WebsocketEndpoint::setOnFail( string_cb on_fail ) { m_on_fail = std::move( on_fail ); } +void WebsocketEndpoint::setOnMessage( char_cb on_message ) { m_on_message = std::move( on_message ); } +void WebsocketEndpoint::setOnClose( void_cb on_close ) { m_on_close = std::move( on_close ); } +void WebsocketEndpoint::setOnSendFail( string_cb on_send_fail ) { m_on_send_fail = std::move( on_send_fail ); } void WebsocketEndpoint::run() { m_ioc.reset(); - if(m_properties.get("use_ssl")) { - ssl::context ctx{ssl::context::sslv23}; - ctx.set_verify_mode(ssl::context::verify_peer ); + if( m_properties.get( "use_ssl" ) ) + { + ssl::context ctx{ ssl::context::sslv23 }; + ctx.set_verify_mode( ssl::context::verify_peer ); ctx.set_default_verify_paths(); - m_session = new WebsocketSessionTLS( - m_ioc, - ctx, - &m_properties, - m_on_open, - m_on_fail, - m_on_message, - m_on_close, - m_on_send_fail - ); - } else { - m_session = new WebsocketSessionNoTLS( - m_ioc, - &m_properties, - m_on_open, - m_on_fail, - m_on_message, - m_on_close, - m_on_send_fail - ); + m_session = new WebsocketSessionTLS( m_ioc, ctx, &m_properties, m_on_open, m_on_fail, m_on_message, m_on_close, + m_on_send_fail ); + } + else + { + m_session = new WebsocketSessionNoTLS( m_ioc, &m_properties, m_on_open, m_on_fail, m_on_message, m_on_close, + m_on_send_fail ); } m_session->run(); @@ -54,18 +36,18 @@ void WebsocketEndpoint::run() } void WebsocketEndpoint::stop() -{ +{ m_ioc.stop(); - if(m_session) m_session->stop(); + if( m_session ) + m_session->stop(); } +csp::Dictionary & WebsocketEndpoint::getProperties() { return m_properties; } -csp::Dictionary& WebsocketEndpoint::getProperties() { - return m_properties; +void WebsocketEndpoint::send( const std::string & s ) +{ + if( m_session ) + m_session->send( s ); } -void WebsocketEndpoint::send(const std::string& s) -{ if(m_session) m_session->send(s); } - - -} \ No newline at end of file +} // namespace csp::adapters::websocket \ No newline at end of file diff --git a/cpp/csp/core/Exception.cpp b/cpp/csp/core/Exception.cpp index e4480754..bb02a844 100644 --- a/cpp/csp/core/Exception.cpp +++ b/cpp/csp/core/Exception.cpp @@ -4,22 +4,22 @@ #include #include -#include #include -#include +#include #include +#include #ifndef WIN32 #include #endif -//From https://stackoverflow.com/questions/2443135/how-do-i-find-where-an-exception-was-thrown-in-c/2443366 +// From https://stackoverflow.com/questions/2443135/how-do-i-find-where-an-exception-was-thrown-in-c/2443366 static void csp_terminate( void ); static bool set_sigabrt_handler(); // invoke set_terminate as part of global constant initialization // This for uncaught exceptions -static const bool SET_TERMINATE = std::set_terminate( csp_terminate ); +static const bool SET_TERMINATE = std::set_terminate( csp_terminate ); static const bool SET_SIGABRT_HANDLER = set_sigabrt_handler(); static void printBacktrace( char ** messages, int size, std::ostream & dest ) @@ -34,31 +34,30 @@ static void printBacktrace( char ** messages, int size, std::ostream & dest ) { char *begin_name = 0, *begin_offset = 0; char tmp[1024]; - strncpy( tmp, messages[i], sizeof(tmp) ); - tmp[ sizeof( tmp ) - 1 ] = 0; + strncpy( tmp, messages[i], sizeof( tmp ) ); + tmp[sizeof( tmp ) - 1] = 0; // find parentheses and +address offset surrounding the mangled name: // ./module(function+0x15c) [0x8048a6d] - for (char *p = tmp; *p; ++p) + for( char * p = tmp; *p; ++p ) { - if (*p == '(') + if( *p == '(' ) begin_name = p; - else if (*p == '+') + else if( *p == '+' ) { begin_offset = p; break; } } - if (begin_name && begin_offset - && begin_name < begin_offset) + if( begin_name && begin_offset && begin_name < begin_offset ) { begin_name++; *begin_offset = '\0'; #ifndef WIN32 int status; - char* demangled = abi::__cxa_demangle(begin_name, NULL, NULL, &status); + char * demangled = abi::__cxa_demangle( begin_name, NULL, NULL, &status ); dest << "[bt]: (" << i << ") " << ( status == 0 ? demangled : messages[i] ) << std::endl; free( demangled ); #else @@ -67,7 +66,7 @@ static void printBacktrace( char ** messages, int size, std::ostream & dest ) } else { - dest << "[bt]: (" << i << ") " << messages[i] << std::endl; + dest << "[bt]: (" << i << ") " << messages[i] << std::endl; } } @@ -76,10 +75,10 @@ static void printBacktrace( char ** messages, int size, std::ostream & dest ) void printBacktrace() { -//TODO get stack traces on windows +// TODO get stack traces on windows #ifndef WIN32 - void *array[50]; - int size = backtrace( array, 50 ); + void * array[50]; + int size = backtrace( array, 50 ); auto messages = backtrace_symbols( array, size ); printBacktrace( messages, size, std::cerr ); free( messages ); @@ -90,28 +89,25 @@ void csp_terminate() { static int tried_throw = 0; - try + try { // try once to re-throw currently active exception - if( !tried_throw++ ) + if( !tried_throw++ ) throw; } catch( const csp::Exception & ex ) { - std::cerr << __FUNCTION__ << " caught unhandled csp::Exception. what(): " - << ex.what() << std::endl; + std::cerr << __FUNCTION__ << " caught unhandled csp::Exception. what(): " << ex.what() << std::endl; if( ex.btsize() > 0 ) printBacktrace( ex.btmessages(), ex.btsize(), std::cerr ); } catch( const std::exception & e ) { - std::cerr << __FUNCTION__ << " caught unhandled std::exception. what(): " - << e.what() << std::endl; + std::cerr << __FUNCTION__ << " caught unhandled std::exception. what(): " << e.what() << std::endl; } - catch( ... ) + catch( ... ) { - std::cerr << __FUNCTION__ << " caught unknown/unhandled exception." - << std::endl; + std::cerr << __FUNCTION__ << " caught unknown/unhandled exception." << std::endl; } printBacktrace(); @@ -122,19 +118,18 @@ void csp_terminate() abort(); } -//This is for coredumps +// This is for coredumps #ifndef WIN32 -void sigabrt_handler(int sig_num, siginfo_t* info, void* ctx) +void sigabrt_handler( int sig_num, siginfo_t * info, void * ctx ) { - std::cerr << "signal " << sig_num - << " (" << strsignal(sig_num) << "), address is " - << info -> si_addr << " from " << std::endl; + std::cerr << "signal " << sig_num << " (" << strsignal( sig_num ) << "), address is " << info->si_addr << " from " + << std::endl; printBacktrace(); - signal(SIGABRT, SIG_DFL); - signal(SIGSEGV, SIG_DFL); - signal(SIGBUS, SIG_DFL); + signal( SIGABRT, SIG_DFL ); + signal( SIGSEGV, SIG_DFL ); + signal( SIGBUS, SIG_DFL ); abort(); } @@ -144,28 +139,28 @@ bool set_sigabrt_handler() static struct sigaction sigact; sigact.sa_sigaction = sigabrt_handler; - sigact.sa_flags = SA_RESTART | SA_SIGINFO; + sigact.sa_flags = SA_RESTART | SA_SIGINFO; - sigaction(SIGABRT, &sigact, NULL); - sigaction(SIGSEGV, &sigact, NULL); - sigaction(SIGBUS, &sigact, NULL); + sigaction( SIGABRT, &sigact, NULL ); + sigaction( SIGSEGV, &sigact, NULL ); + sigaction( SIGBUS, &sigact, NULL ); return true; } #else -void sigabrt_handler(int sig_num) +void sigabrt_handler( int sig_num ) { std::cerr << "signal " << sig_num << " from " << std::endl; printBacktrace(); - signal(SIGABRT, SIG_DFL); - signal(SIGSEGV, SIG_DFL); + signal( SIGABRT, SIG_DFL ); + signal( SIGSEGV, SIG_DFL ); abort(); } bool set_sigabrt_handler() { - signal(SIGABRT, sigabrt_handler); - signal(SIGSEGV, sigabrt_handler); + signal( SIGABRT, sigabrt_handler ); + signal( SIGSEGV, sigabrt_handler ); return true; } #endif @@ -173,50 +168,51 @@ bool set_sigabrt_handler() void csp::Exception::setbt() { #ifndef WIN32 - void *array[50]; - m_backtracesize = backtrace( array, 50 ); - char **messages = backtrace_symbols( array, m_backtracesize ); + void * array[50]; + m_backtracesize = backtrace( array, 50 ); + char ** messages = backtrace_symbols( array, m_backtracesize ); m_backtracemessages = messages; #endif } -static char ** dupe_backtraces( char** bt, int n ) +static char ** dupe_backtraces( char ** bt, int n ) { - if( bt == nullptr ) return nullptr; + if( bt == nullptr ) + return nullptr; size_t len = n * sizeof( char * ); for( int i = 0; i < n; ++i ) - len += strlen( bt[ i ] ) + 1; - char ** newbt = (char **) malloc( len ); + len += strlen( bt[i] ) + 1; + char ** newbt = (char **)malloc( len ); memcpy( newbt, bt, len ); for( int i = 0; i < n; ++i ) - newbt[i] = (char *)( newbt + ( ( (char **) bt[ i ] ) - bt ) ); + newbt[i] = (char *)( newbt + ( ( (char **)bt[i] ) - bt ) ); return newbt; } -csp::Exception::Exception( const csp::Exception &orig ) : - m_full( orig.m_full ), - m_exType( orig.m_exType ), - m_description( orig.m_description ), - m_file( orig.m_file ), - m_function( orig.m_function ), - m_line( orig.m_line ), - m_backtracesize( orig.m_backtracesize ), - m_backtracemessages( dupe_backtraces( orig.m_backtracemessages, orig.m_backtracesize ) ) +csp::Exception::Exception( const csp::Exception & orig ) + : m_full( orig.m_full ) + , m_exType( orig.m_exType ) + , m_description( orig.m_description ) + , m_file( orig.m_file ) + , m_function( orig.m_function ) + , m_line( orig.m_line ) + , m_backtracesize( orig.m_backtracesize ) + , m_backtracemessages( dupe_backtraces( orig.m_backtracemessages, orig.m_backtracesize ) ) { } -csp::Exception::Exception( csp::Exception && donor ) : - m_full( std::move( donor.m_full ) ), - m_exType( std::move( donor.m_exType ) ), - m_description( std::move( donor.m_description ) ), - m_file( std::move( donor.m_file ) ), - m_function( std::move( donor.m_function ) ), - m_line( donor.m_line ), - m_backtracesize( donor.m_backtracesize ), - m_backtracemessages( donor.m_backtracemessages ) +csp::Exception::Exception( csp::Exception && donor ) + : m_full( std::move( donor.m_full ) ) + , m_exType( std::move( donor.m_exType ) ) + , m_description( std::move( donor.m_description ) ) + , m_file( std::move( donor.m_file ) ) + , m_function( std::move( donor.m_function ) ) + , m_line( donor.m_line ) + , m_backtracesize( donor.m_backtracesize ) + , m_backtracemessages( donor.m_backtracemessages ) { donor.m_backtracemessages = nullptr; } diff --git a/cpp/csp/core/Time.cpp b/cpp/csp/core/Time.cpp index cb7055ca..c663b66f 100644 --- a/cpp/csp/core/Time.cpp +++ b/cpp/csp/core/Time.cpp @@ -4,31 +4,28 @@ namespace csp { // XXXXXX -//This Code was taken from glibc time/offtime.c, the reason being that gmtime_r actually calls the global tzlock +// This Code was taken from glibc time/offtime.c, the reason being that gmtime_r actually calls the global tzlock //(even though technically we dont need tz here ). This makes it lockless -static const unsigned short int __mon_yday[2][13] = - { +static const unsigned short int __mon_yday[2][13] = { /* Normal years. */ { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 }, /* Leap years. */ - { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } - }; - + { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } }; tm DateTime::asTM() const { - tm TM{0}; + tm TM{ 0 }; time_t days, rem, y; - const unsigned short int *ip; + const unsigned short int * ip; days = m_ticks / NANOS_PER_DAY; - rem = m_ticks % NANOS_PER_DAY; - while (rem < 0) + rem = m_ticks % NANOS_PER_DAY; + while( rem < 0 ) { rem += NANOS_PER_DAY; --days; } - while (rem >= NANOS_PER_DAY) + while( rem >= NANOS_PER_DAY ) { rem -= NANOS_PER_DAY; ++days; @@ -38,40 +35,38 @@ tm DateTime::asTM() const TM.tm_min = rem / ( 60 * NANOS_PER_SECOND ); TM.tm_sec = ( rem % ( 60 * NANOS_PER_SECOND ) ) / NANOS_PER_SECOND; /* January 1, 1970 was a Thursday. */ - TM.tm_wday = (4 + days) % 7; - if (TM.tm_wday < 0) + TM.tm_wday = ( 4 + days ) % 7; + if( TM.tm_wday < 0 ) TM.tm_wday += 7; y = 1970; -#define DIV(a, b) ((a) / (b) - ((a) % (b) < 0)) -#define LEAPS_THRU_END_OF(y) (DIV (y, 4) - DIV (y, 100) + DIV (y, 400)) -#define __isleap(year) \ - ((year) % 4 == 0 && ((year) % 100 != 0 || (year) % 400 == 0)) - - while (days < 0 || days >= (__isleap(y) ? 366 : 365)) +#define DIV( a, b ) ( ( a ) / ( b ) - ( ( a ) % ( b ) < 0 ) ) +#define LEAPS_THRU_END_OF( y ) ( DIV( y, 4 ) - DIV( y, 100 ) + DIV( y, 400 ) ) +#ifndef __isleap +#define __isleap( year ) ( ( year ) % 4 == 0 && ( ( year ) % 100 != 0 || ( year ) % 400 == 0 ) ) +#endif + while( days < 0 || days >= ( __isleap( y ) ? 366 : 365 ) ) { /* Guess a corrected year, assuming 365 days per year. */ - time_t yg = y + days / 365 - (days % 365 < 0); + time_t yg = y + days / 365 - ( days % 365 < 0 ); /* Adjust DAYS and Y to match the guessed year. */ - days -= ((yg - y) * 365 - + LEAPS_THRU_END_OF (yg - 1) - - LEAPS_THRU_END_OF (y - 1)); + days -= ( ( yg - y ) * 365 + LEAPS_THRU_END_OF( yg - 1 ) - LEAPS_THRU_END_OF( y - 1 ) ); y = yg; } TM.tm_year = y - 1900; - if (TM.tm_year != y - 1900) + if( TM.tm_year != y - 1900 ) CSP_THROW( RuntimeException, "Failed to convert DateTime to struct tm: year overflow" ); TM.tm_yday = days; - ip = __mon_yday[__isleap(y)]; - for (y = 11; days < (long int) ip[y]; --y) + ip = __mon_yday[__isleap( y )]; + for( y = 11; days < (long int)ip[y]; --y ) continue; days -= ip[y]; - TM.tm_mon = y; + TM.tm_mon = y; TM.tm_mday = days + 1; return TM; } -//XXXX +// XXXX -}; +}; // namespace csp diff --git a/cpp/csp/cppnodes/baselibimpl.cpp b/cpp/csp/cppnodes/baselibimpl.cpp index 52a5537d..79c5db4e 100644 --- a/cpp/csp/cppnodes/baselibimpl.cpp +++ b/cpp/csp/cppnodes/baselibimpl.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include namespace csp::cppnodes @@ -12,18 +12,14 @@ def sample(trigger: ts['Y'], x: ts['T']): */ DECLARE_CPPNODE( sample ) { - INIT_CPPNODE( sample ) - {} + INIT_CPPNODE( sample ) {} TS_INPUT( Generic, trigger ); TS_INPUT( Generic, x ); TS_OUTPUT( Generic ); - START() - { - csp.make_passive( x ); - } + START() { csp.make_passive( x ); } INVOKE() { @@ -42,16 +38,14 @@ def firstN(x: ts['T'], N: int): DECLARE_CPPNODE( firstN ) { - INIT_CPPNODE( firstN ) - { - } + INIT_CPPNODE( firstN ) {} - TS_INPUT( Generic, x ); + TS_INPUT( Generic, x ); SCALAR_INPUT( int64_t, N ); TS_OUTPUT( Generic ); - STATE_VAR( int, s_count{0} ); + STATE_VAR( int, s_count{ 0 } ); START() { @@ -77,16 +71,12 @@ def count(x: ts['T']): */ DECLARE_CPPNODE( count ) { - INIT_CPPNODE( count ) - {} + INIT_CPPNODE( count ) {} TS_INPUT( Generic, x ); TS_OUTPUT( int64_t ); - INVOKE() - { - RETURN( csp.count( x ) ); - } + INVOKE() { RETURN( csp.count( x ) ); } }; EXPORT_CPPNODE( count ); @@ -99,8 +89,7 @@ def _delay_by_timedelta(x: ts['T'], delay: timedelta): */ DECLARE_CPPNODE( _delay_by_timedelta ) { - INIT_CPPNODE( _delay_by_timedelta ) - {} + INIT_CPPNODE( _delay_by_timedelta ) {} TS_INPUT( Generic, x ); SCALAR_INPUT( TimeDelta, delay ); @@ -132,8 +121,7 @@ DECLARE_CPPNODE( _delay_by_ticks ) TS_OUTPUT( Generic ); - INIT_CPPNODE( _delay_by_ticks ) - {} + INIT_CPPNODE( _delay_by_ticks ) {} START() { @@ -147,11 +135,12 @@ DECLARE_CPPNODE( _delay_by_ticks ) { if( csp.ticked( x ) && csp.count( x ) > delay ) { - switchCspType( x.type(), [this]( auto tag ) - { - using ElemT = typename decltype(tag)::type; - RETURN( x.valueAtIndex( delay ) ); - } ); + switchCspType( x.type(), + [this]( auto tag ) + { + using ElemT = typename decltype( tag )::type; + RETURN( x.valueAtIndex( delay ) ); + } ); } } }; @@ -166,8 +155,7 @@ def merge(x: ts['T'], y : ts[ 'T' ] ): */ DECLARE_CPPNODE( merge ) { - INIT_CPPNODE( merge ) - {} + INIT_CPPNODE( merge ) {} TS_INPUT( Generic, x ); TS_INPUT( Generic, y ); @@ -185,7 +173,6 @@ DECLARE_CPPNODE( merge ) EXPORT_CPPNODE( merge ); - /* @csp.node(cppimpl=_cspbaselibimpl.split) def split(flag: ts[bool], x: ts['T']): @@ -194,18 +181,14 @@ def split(flag: ts[bool], x: ts['T']): */ DECLARE_CPPNODE( split ) { - INIT_CPPNODE( split ) - {} + INIT_CPPNODE( split ) {} - TS_INPUT( bool, flag ); + TS_INPUT( bool, flag ); TS_INPUT( Generic, x ); TS_NAMED_OUTPUT_RENAMED( Generic, false, false_ ); - TS_NAMED_OUTPUT_RENAMED( Generic, true, true_ ); + TS_NAMED_OUTPUT_RENAMED( Generic, true, true_ ); - START() - { - csp.make_passive( flag ); - } + START() { csp.make_passive( flag ); } INVOKE() { @@ -228,8 +211,7 @@ def cast_int_to_float(x: csp.ts[int]): */ DECLARE_CPPNODE( cast_int_to_float ) { - INIT_CPPNODE( cast_int_to_float ) - {} + INIT_CPPNODE( cast_int_to_float ) {} TS_INPUT( int64_t, x ); TS_OUTPUT( double ); @@ -250,17 +232,13 @@ def filter(flag: ts[bool], x: ts['T']): */ DECLARE_CPPNODE( filter ) { - INIT_CPPNODE( filter ) - {} + INIT_CPPNODE( filter ) {} - TS_INPUT( bool, flag ); + TS_INPUT( bool, flag ); TS_INPUT( Generic, x ); TS_OUTPUT( Generic ); - START() - { - csp.make_passive( flag ); - } + START() { csp.make_passive( flag ); } INVOKE() { @@ -278,8 +256,7 @@ def _drop_dups_float(x: ts[float], eps: float): */ DECLARE_CPPNODE( _drop_dups_float ) { - INIT_CPPNODE( _drop_dups_float ) - {} + INIT_CPPNODE( _drop_dups_float ) {} TS_INPUT( double, x ); @@ -287,17 +264,17 @@ DECLARE_CPPNODE( _drop_dups_float ) TS_OUTPUT( double ); - STATE_VAR( bool, s_first{true} ); + STATE_VAR( bool, s_first{ true } ); STATE_VAR( double, s_prev{} ); INVOKE() { if( csp.ticked( x ) ) { - if( s_first || ( isnan( x ) != isnan( s_prev ) ) || ( !isnan( x ) && fabs( x - s_prev ) >= eps )) + if( s_first || ( isnan( x ) != isnan( s_prev ) ) || ( !isnan( x ) && fabs( x - s_prev ) >= eps ) ) { s_first = false; - s_prev = x; + s_prev = x; RETURN( x ); } } @@ -336,9 +313,9 @@ def unroll(x: ts[['T']]): */ DECLARE_CPPNODE( unroll ) { - TS_INPUT( Generic, x ); - ALARM( Generic, alarm ); - STATE_VAR( uint32_t, s_pending{0} ); + TS_INPUT( Generic, x ); + ALARM( Generic, alarm ); + STATE_VAR( uint32_t, s_pending{ 0 } ); TS_OUTPUT( Generic ); @@ -346,54 +323,55 @@ DECLARE_CPPNODE( unroll ) INIT_CPPNODE( unroll ) { - //we need to access type information using the input / outout defs because the actual - //ts() instances arent created at this point + // we need to access type information using the input / outout defs because the actual + // ts() instances arent created at this point auto & x_def = tsinputDef( "x" ); - if( x_def.type -> type() != CspType::Type::ARRAY ) - CSP_THROW( TypeError, "unroll expected ts array type, got " << x_def.type -> type() ); + if( x_def.type->type() != CspType::Type::ARRAY ) + CSP_THROW( TypeError, "unroll expected ts array type, got " << x_def.type->type() ); auto * aType = static_cast( x_def.type.get() ); - elemType = aType -> elemType(); + elemType = aType->elemType(); - //we cant easily support unrolling list of typed lists ( ts[ [[int]] ] ). Since we dont recurse type info more than one level - //the input elemType would be DIALECT_GENERIC, but the output type would be the correct ARRAY:Type type. Briding the two here is - //complex and not (currently) wirht the effort, so fallback to python by throwing NotImplemented + // we cant easily support unrolling list of typed lists ( ts[ [[int]] ] ). Since we dont recurse type info more + // than one level the input elemType would be DIALECT_GENERIC, but the output type would be the correct + // ARRAY:Type type. Briding the two here is complex and not (currently) wirht the effort, so fallback to python + // by throwing NotImplemented auto & out_def = tsoutputDef( "" ); - if( out_def.type -> type() == CspType::Type::ARRAY ) + if( out_def.type->type() == CspType::Type::ARRAY ) CSP_THROW( NotImplemented, "unroll cppimpl doesnt currently support unrolloing lists of typed lists" ); } INVOKE() { - //single switch up front, no need to do it multiple times - switchCspType( elemType, [this]( auto tag ) - { - using ElemT = typename decltype(tag)::type; - using ArrayT = typename CspType::Type::toCArrayType::type; - - if( csp.ticked( x ) ) - { - auto & v = x.lastValue(); - size_t sz = v.size(); - if( likely( sz > 0 ) ) - { - size_t idx = 0; - if( !s_pending ) - CSP_OUTPUT( static_cast( v[idx++] ) ); - - s_pending += sz - idx; - for( ; idx < sz; ++idx ) - csp.schedule_alarm( alarm, TimeDelta::ZERO(), static_cast( v[idx] ) ); - } - } + // single switch up front, no need to do it multiple times + switchCspType( elemType, + [this]( auto tag ) + { + using ElemT = typename decltype( tag )::type; + using ArrayT = typename CspType::Type::toCArrayType::type; - if( csp.ticked( alarm ) ) - { - --s_pending; - RETURN( alarm.lastValue() ); - } + if( csp.ticked( x ) ) + { + auto & v = x.lastValue(); + size_t sz = v.size(); + if( likely( sz > 0 ) ) + { + size_t idx = 0; + if( !s_pending ) + CSP_OUTPUT( static_cast( v[idx++] ) ); + + s_pending += sz - idx; + for( ; idx < sz; ++idx ) + csp.schedule_alarm( alarm, TimeDelta::ZERO(), static_cast( v[idx] ) ); + } + } - } ); + if( csp.ticked( alarm ) ) + { + --s_pending; + RETURN( alarm.lastValue() ); + } + } ); } }; @@ -414,45 +392,47 @@ DECLARE_CPPNODE( collect ) INIT_CPPNODE( collect ) { - //we cant process 'T' of type typed list ( it [int] ) because the input type would be ARRAY:INT64 - //but the output type would become ARRAY:DIALECT_GENERIC, which we cant create. just fallback to python + // we cant process 'T' of type typed list ( it [int] ) because the input type would be ARRAY:INT64 + // but the output type would become ARRAY:DIALECT_GENERIC, which we cant create. just fallback to python auto & x_def = tsinputDef( "x" ); - if( x_def.type -> type() == CspType::Type::ARRAY ) + if( x_def.type->type() == CspType::Type::ARRAY ) CSP_THROW( NotImplemented, "cppimpl of collect cannot handle typed lists inputs" ); auto & out_def = tsoutputDef( "" ); - if( out_def.type -> type() != CspType::Type::ARRAY ) - CSP_THROW( TypeError, "cppimpl for collect expected output type to be list, got " << out_def.type -> type() ); + if( out_def.type->type() != CspType::Type::ARRAY ) + CSP_THROW( TypeError, "cppimpl for collect expected output type to be list, got " << out_def.type->type() ); auto * aType = static_cast( out_def.type.get() ); - elemType = aType -> elemType(); + elemType = aType->elemType(); - if( elemType -> type() != x_def.type -> type() ) - CSP_THROW( TypeError, "cppimpl for collect has unexpected type mistmatch, input type is " << x_def.type -> type () << - " but output array type is " << elemType -> type() ); + if( elemType->type() != x_def.type->type() ) + CSP_THROW( TypeError, + "cppimpl for collect has unexpected type mistmatch, input type is " + << x_def.type->type() << " but output array type is " << elemType->type() ); } START() { - //to avoid the need to check in every invoke + // to avoid the need to check in every invoke if( x.size() == 0 ) csp.make_passive( x ); } INVOKE() { - //single switch up front, no need to do it multiple times - //we expect all elements to be of the same type - switchCspType( elemType, [this]( auto tag ) + // single switch up front, no need to do it multiple times + // we expect all elements to be of the same type + switchCspType( elemType, + [this]( auto tag ) { - using TagType = decltype( tag ); - using ElemT = typename TagType::type; - using ArrayT = typename CspType::Type::toCArrayType::type; - - ArrayT & out = unnamed_output().reserveSpace(); - out.clear(); - for( auto it = x.tickedinputs(); it; ++it ) - out.emplace_back( it -> lastValueTyped() ); + using TagType = decltype( tag ); + using ElemT = typename TagType::type; + using ArrayT = typename CspType::Type::toCArrayType::type; + + ArrayT & out = unnamed_output().reserveSpace(); + out.clear(); + for( auto it = x.tickedinputs(); it; ++it ) + out.emplace_back( it->lastValueTyped() ); } ); } }; @@ -475,21 +455,18 @@ DECLARE_CPPNODE( demultiplex ) INIT_CPPNODE( demultiplex ) { auto & key_def = tsinputDef( "key" ); - if( key_def.type -> type() != CspType::Type::STRING ) + if( key_def.type->type() != CspType::Type::STRING ) CSP_THROW( NotImplemented, "cppimpl for demultiplex not supported on non-string key types" ); } - START() - { - csp.make_passive( key ); - } + START() { csp.make_passive( key ); } INVOKE() { if( csp.valid( key ) ) { - auto &key_str = key.lastValue(); - auto elemId = unnamed_output().elemId( key_str ); + auto & key_str = key.lastValue(); + auto elemId = unnamed_output().elemId( key_str ); if( elemId != InputId::ELEM_ID_NONE ) { unnamed_output()[elemId].output( x ); @@ -517,12 +494,12 @@ DECLARE_CPPNODE( multiplex ) TS_OUTPUT( Generic ); - STATE_VAR( bool, s_key_valid{false} ); + STATE_VAR( bool, s_key_valid{ false } ); INIT_CPPNODE( multiplex ) { auto & key_def = tsinputDef( "key" ); - if( key_def.type -> type() != CspType::Type::STRING ) + if( key_def.type->type() != CspType::Type::STRING ) CSP_THROW( NotImplemented, "cppimpl for multiplex not supported on non-string key types" ); } @@ -530,7 +507,7 @@ DECLARE_CPPNODE( multiplex ) { if( csp.ticked( key ) ) { - auto &key_str = key.lastValue(); + auto & key_str = key.lastValue(); csp.make_passive( x ); int64_t elemId = x.elemId( key_str ); @@ -549,10 +526,9 @@ DECLARE_CPPNODE( multiplex ) if( s_key_valid ) { - auto &key_str = key.lastValue(); + auto & key_str = key.lastValue(); int64_t elemId = x.elemId( key_str ); - if( csp.ticked( x[elemId] ) || - ( tick_on_index && csp.ticked(key) && csp.valid(x[elemId]) ) ) + if( csp.ticked( x[elemId] ) || ( tick_on_index && csp.ticked( key ) && csp.valid( x[elemId] ) ) ) { CSP_OUTPUT( x[elemId] ); } @@ -576,12 +552,9 @@ DECLARE_CPPNODE( times ) TS_INPUT( Generic, x ); TS_OUTPUT( DateTime ); - INIT_CPPNODE( times ) { } + INIT_CPPNODE( times ) {} - INVOKE() - { - RETURN( now() ); - } + INVOKE() { RETURN( now() ); } }; EXPORT_CPPNODE( times ); @@ -600,12 +573,9 @@ DECLARE_CPPNODE( times_ns ) TS_INPUT( Generic, x ); TS_OUTPUT( int64_t ); - INIT_CPPNODE( times_ns ) { } + INIT_CPPNODE( times_ns ) {} - INVOKE() - { - RETURN( now().asNanoseconds() ); - } + INVOKE() { RETURN( now().asNanoseconds() ); } }; EXPORT_CPPNODE( times_ns ); @@ -617,10 +587,9 @@ def struct_field(x: ts['T'], field: str, fieldType: 'Y'): */ DECLARE_CPPNODE( struct_field ) { - INIT_CPPNODE( struct_field ) - {} + INIT_CPPNODE( struct_field ) {} - TS_INPUT( StructPtr, x ); + TS_INPUT( StructPtr, x ); SCALAR_INPUT( std::string, field ); TS_OUTPUT( Generic ); @@ -628,19 +597,20 @@ DECLARE_CPPNODE( struct_field ) START() { auto * structType = static_cast( x.type() ); - m_fieldAccess = structType -> meta() -> field( field ); + m_fieldAccess = structType->meta()->field( field ); if( !m_fieldAccess ) - CSP_THROW( TypeError, "Struct " << structType -> meta() -> name() << " has no field named " << field.value() ); + CSP_THROW( TypeError, "Struct " << structType->meta()->name() << " has no field named " << field.value() ); } INVOKE() { - if( m_fieldAccess -> isSet( x.lastValue().get() ) ) + if( m_fieldAccess->isSet( x.lastValue().get() ) ) { - switchCspType( m_fieldAccess -> type(), [this]( auto tag ) + switchCspType( m_fieldAccess->type(), + [this]( auto tag ) { - using T = typename decltype(tag)::type; - RETURN( m_fieldAccess -> value( x.lastValue().get() ) ); + using T = typename decltype( tag )::type; + RETURN( m_fieldAccess->value( x.lastValue().get() ) ); } ); } } @@ -651,8 +621,8 @@ DECLARE_CPPNODE( struct_field ) EXPORT_CPPNODE( struct_field ); -//fromts and collectts are unfortunately identical except for tickeditems() vs validitems() -//but i dont think its enough to warrant any refactoring at the moment +// fromts and collectts are unfortunately identical except for tickeditems() vs validitems() +// but i dont think its enough to warrant any refactoring at the moment /* @csp.node def struct_fromts(cls: 'T', inputs: {str: ts[object]}): @@ -661,28 +631,29 @@ def struct_fromts(cls: 'T', inputs: {str: ts[object]}): DECLARE_CPPNODE( struct_fromts ) { TS_DICTBASKET_INPUT( Generic, inputs ); - TS_INPUT( Generic, trigger ); - SCALAR_INPUT( StructMetaPtr, cls ); - SCALAR_INPUT( bool, use_trigger ); + TS_INPUT( Generic, trigger ); + SCALAR_INPUT( StructMetaPtr, cls ); + SCALAR_INPUT( bool, use_trigger ); TS_OUTPUT( StructPtr ); - INIT_CPPNODE( struct_fromts ) - { - } + INIT_CPPNODE( struct_fromts ) {} START() { for( size_t elemId = 0; elemId < inputs.shape().size(); ++elemId ) { - auto & key = inputs.shape()[ elemId ]; - auto & structField = cls.value() -> field( key ); + auto & key = inputs.shape()[elemId]; + auto & structField = cls.value()->field( key ); if( !structField ) - CSP_THROW( ValueError, cls.value() -> name() << ".fromts() received unknown struct field \"" << key << "\"" ); + CSP_THROW( ValueError, + cls.value()->name() << ".fromts() received unknown struct field \"" << key << "\"" ); - if( structField -> type() -> type() != inputs[ elemId ].type() -> type() ) - CSP_THROW( TypeError, cls.value() -> name() << ".fromts() field \"" << key << "\" expected ts type " - << structField -> type() -> type() << " but got " << inputs[ elemId ].type() -> type() ); + if( structField->type()->type() != inputs[elemId].type()->type() ) + CSP_THROW( TypeError, + cls.value()->name() + << ".fromts() field \"" << key << "\" expected ts type " << structField->type()->type() + << " but got " << inputs[elemId].type()->type() ); m_structFields.push_back( structField.get() ); } @@ -693,16 +664,16 @@ DECLARE_CPPNODE( struct_fromts ) INVOKE() { - auto out = cls.value() -> create(); + auto out = cls.value()->create(); for( auto it = inputs.validinputs(); it; ++it ) { auto * fieldAccess = m_structFields[it.elemId()]; - switchCspType( it -> type(), [&it,&out,fieldAccess]( auto tag ) + switchCspType( it->type(), + [&it, &out, fieldAccess]( auto tag ) { - using ElemT = typename decltype(tag)::type; - fieldAccess -> setValue( out.get(), it -> lastValueTyped() ); - } - ); + using ElemT = typename decltype( tag )::type; + fieldAccess->setValue( out.get(), it->lastValueTyped() ); + } ); } CSP_OUTPUT( std::move( out ) ); @@ -724,22 +695,23 @@ DECLARE_CPPNODE( struct_collectts ) SCALAR_INPUT( StructMetaPtr, cls ); TS_OUTPUT( StructPtr ); - INIT_CPPNODE( struct_collectts ) - { - } + INIT_CPPNODE( struct_collectts ) {} START() { for( size_t elemId = 0; elemId < inputs.shape().size(); ++elemId ) { - auto & key = inputs.shape()[ elemId ]; - auto & structField = cls.value() -> field( key ); + auto & key = inputs.shape()[elemId]; + auto & structField = cls.value()->field( key ); if( !structField ) - CSP_THROW( ValueError, cls.value() -> name() << ".collectts() received unknown struct field \"" << key << "\"" ); + CSP_THROW( ValueError, + cls.value()->name() << ".collectts() received unknown struct field \"" << key << "\"" ); - if( structField -> type() -> type() != inputs[ elemId ].type() -> type() ) - CSP_THROW( TypeError, cls.value() -> name() << ".collectts() field \"" << key << "\" expected ts type " - << structField -> type() -> type() << " but got " << inputs[ elemId ].type() -> type() ); + if( structField->type()->type() != inputs[elemId].type()->type() ) + CSP_THROW( TypeError, + cls.value()->name() + << ".collectts() field \"" << key << "\" expected ts type " + << structField->type()->type() << " but got " << inputs[elemId].type()->type() ); m_structFields.push_back( structField.get() ); } @@ -747,16 +719,16 @@ DECLARE_CPPNODE( struct_collectts ) INVOKE() { - auto out = cls.value() -> create(); + auto out = cls.value()->create(); for( auto it = inputs.tickedinputs(); it; ++it ) { auto * fieldAccess = m_structFields[it.elemId()]; - switchCspType( it -> type(), [&it,&out,fieldAccess]( auto tag ) + switchCspType( it->type(), + [&it, &out, fieldAccess]( auto tag ) { - using ElemT = typename decltype(tag)::type; - fieldAccess -> setValue( out.get(), it -> lastValueTyped() ); - } - ); + using ElemT = typename decltype( tag )::type; + fieldAccess->setValue( out.get(), it->lastValueTyped() ); + } ); } CSP_OUTPUT( std::move( out ) ); @@ -767,4 +739,4 @@ DECLARE_CPPNODE( struct_collectts ) EXPORT_CPPNODE( struct_collectts ); -} +} // namespace csp::cppnodes diff --git a/cpp/csp/cppnodes/basketlibimpl.cpp b/cpp/csp/cppnodes/basketlibimpl.cpp index 0749a313..b5c6842c 100644 --- a/cpp/csp/cppnodes/basketlibimpl.cpp +++ b/cpp/csp/cppnodes/basketlibimpl.cpp @@ -12,18 +12,15 @@ DECLARE_CPPNODE( _sync_list ) ALARM( bool, a_end ); - STATE_VAR( size_t, s_count{0} ); + STATE_VAR( size_t, s_count{ 0 } ); STATE_VAR( Scheduler::Handle, s_alarm_handle ); STATE_VAR( std::vector, s_current_ticked{} ); TS_LISTBASKET_OUTPUT( Generic ); - INIT_CPPNODE( _sync_list ) { } + INIT_CPPNODE( _sync_list ) {} - START() - { - s_current_ticked.resize( x.size(), false ); - } + START() { s_current_ticked.resize( x.size(), false ); } INVOKE() { @@ -36,11 +33,11 @@ DECLARE_CPPNODE( _sync_list ) for( auto it = x.tickedinputs(); it; ++it ) { - if( s_current_ticked[ it.elemId() ] == false ) + if( s_current_ticked[it.elemId()] == false ) { s_count++; } - s_current_ticked[ it.elemId() ] = true; + s_current_ticked[it.elemId()] = true; } } @@ -52,9 +49,9 @@ DECLARE_CPPNODE( _sync_list ) { for( size_t elemId = 0; elemId < x.size(); elemId++ ) { - if( s_current_ticked[ elemId ] ) + if( s_current_ticked[elemId] ) { - unnamed_output()[ elemId ].output( x[ elemId ] ); + unnamed_output()[elemId].output( x[elemId] ); } } } @@ -85,21 +82,18 @@ DECLARE_CPPNODE( _sample_list ) TS_LISTBASKET_OUTPUT( Generic ); - INIT_CPPNODE( _sample_list ) { } + INIT_CPPNODE( _sample_list ) {} - START() - { - x.makePassive(); - } + START() { x.makePassive(); } INVOKE() { - if( csp.ticked(trigger) ) + if( csp.ticked( trigger ) ) { - for( auto it = x.validinputs(); it; ++it) + for( auto it = x.validinputs(); it; ++it ) { auto idx = it.elemId(); - unnamed_output()[ idx ].output( x[idx] ); + unnamed_output()[idx].output( x[idx] ); } } } @@ -107,4 +101,4 @@ DECLARE_CPPNODE( _sample_list ) EXPORT_CPPNODE( _sample_list ); -} \ No newline at end of file +} // namespace csp::cppnodes \ No newline at end of file diff --git a/cpp/csp/cppnodes/mathimpl.cpp b/cpp/csp/cppnodes/mathimpl.cpp index eeb865f8..200cbe00 100644 --- a/cpp/csp/cppnodes/mathimpl.cpp +++ b/cpp/csp/cppnodes/mathimpl.cpp @@ -1,6 +1,5 @@ -#include #include - +#include namespace csp::cppnodes { @@ -11,112 +10,194 @@ Math operations // Unary operation -template +template DECLARE_CPPNODE( _unary_op ) { TS_INPUT( ArgT, x ); TS_OUTPUT( OutT ); - //Expanded out INIT_CPPNODE without create call... + // Expanded out INIT_CPPNODE without create call... CSP csp; const char * name() const override { return "_unary_op"; } public: _STATIC_CREATE_METHOD( SINGLE_ARG( _unary_op ) ); - _unary_op( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} - - INVOKE() + _unary_op( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) { - RETURN( Func( x ) ); } + + INVOKE() { RETURN( Func( x ) ); } }; -template inline T _abs( T x ){ return std::abs( x ); } -template inline double _ln( T x ){ return std::log( x ); } -template inline double _log2( T x ){ return std::log2( x ); } -template inline double _log10( T x ){ return std::log10( x ); } -template inline double _exp( T x ){ return std::exp( x ); } -template inline double _exp2( T x ){ return std::exp2( x ); } -template inline double _sqrt( T x ){ return std::sqrt( x ); } -template inline double _erf( T x ){ return std::erf( x ); } -template inline double _sin( T x ){ return std::sin( x ); } -template inline double _cos( T x ){ return std::cos( x ); } -template inline double _tan( T x ){ return std::tan( x ); } -template inline double _asin( T x ){ return std::asin( x ); } -template inline double _acos( T x ){ return std::acos( x ); } -template inline double _atan( T x ){ return std::atan( x ); } -template inline double _sinh( T x ){ return std::sinh( x ); } -template inline double _cosh( T x ){ return std::cosh( x ); } -template inline double _tanh( T x ){ return std::tanh( x ); } -template inline double _asinh( T x ){ return std::asinh( x ); } -template inline double _acosh( T x ){ return std::acosh( x ); } -template inline double _atanh( T x ){ return std::atanh( x ); } - -inline bool _not_( bool x ){ return !x; } -inline int64_t _bitwise_not(int64_t x) { return ~x; } - -#define EXPORT_UNARY_OP( Name, ArgType, OutType, Func ) EXPORT_TEMPLATE_CPPNODE( Name, SINGLE_ARG( _unary_op ) ) - EXPORT_UNARY_OP( abs_f, double, double, abs ); - EXPORT_UNARY_OP( abs_i, int64_t, int64_t, abs ); - EXPORT_UNARY_OP( ln_f, double, double, ln ); - EXPORT_UNARY_OP( ln_i, int64_t, double, ln ); - EXPORT_UNARY_OP( log2_f, double, double, log2 ); - EXPORT_UNARY_OP( log2_i, int64_t, double, log2 ); - EXPORT_UNARY_OP( log10_f, double, double, log10 ); - EXPORT_UNARY_OP( log10_i, int64_t, double, log10 ); - EXPORT_UNARY_OP( exp_f, double, double, exp ); - EXPORT_UNARY_OP( exp_i, int64_t, double, exp ); - EXPORT_UNARY_OP( exp2_f, double, double, exp2 ); - EXPORT_UNARY_OP( exp2_i, int64_t, double, exp2 ); - EXPORT_UNARY_OP( sqrt_f, double, double, sqrt ); - EXPORT_UNARY_OP( sqrt_i, int64_t, double, sqrt ); - EXPORT_UNARY_OP( erf_f, double, double, erf ); - EXPORT_UNARY_OP( erf_i, int64_t, double, erf ); - EXPORT_UNARY_OP( sin_f, double, double, sin ); - EXPORT_UNARY_OP( sin_i, int64_t, double, sin ); - EXPORT_UNARY_OP( cos_f, double, double, cos ); - EXPORT_UNARY_OP( cos_i, int64_t, double, cos ); - EXPORT_UNARY_OP( tan_f, double, double, tan ); - EXPORT_UNARY_OP( tan_i, int64_t, double, tan ); - EXPORT_UNARY_OP( asin_f, double, double, asin ); - EXPORT_UNARY_OP( asin_i, int64_t, double, asin ); - EXPORT_UNARY_OP( acos_f, double, double, acos ); - EXPORT_UNARY_OP( acos_i, int64_t, double, acos ); - EXPORT_UNARY_OP( atan_f, double, double, atan ); - EXPORT_UNARY_OP( atan_i, int64_t, double, atan ); - EXPORT_UNARY_OP( sinh_f, double, double, sinh ); - EXPORT_UNARY_OP( sinh_i, int64_t, double, sinh ); - EXPORT_UNARY_OP( cosh_f, double, double, cosh ); - EXPORT_UNARY_OP( cosh_i, int64_t, double, cosh ); - EXPORT_UNARY_OP( tanh_f, double, double, tanh ); - EXPORT_UNARY_OP( tanh_i, int64_t, double, tanh ); - EXPORT_UNARY_OP( asinh_f, double, double, asinh ); - EXPORT_UNARY_OP( asinh_i, int64_t, double, asinh ); - EXPORT_UNARY_OP( acosh_f, double, double, acosh ); - EXPORT_UNARY_OP( acosh_i, int64_t, double, acosh ); - EXPORT_UNARY_OP( atanh_f, double, double, atanh ); - EXPORT_UNARY_OP( atanh_i, int64_t, double, atanh ); - EXPORT_UNARY_OP( not_, bool, bool, not_ ); - EXPORT_UNARY_OP( bitwise_not, int64_t, int64_t, bitwise_not ); +template +inline T _abs( T x ) +{ + return std::abs( x ); +} +template +inline double _ln( T x ) +{ + return std::log( x ); +} +template +inline double _log2( T x ) +{ + return std::log2( x ); +} +template +inline double _log10( T x ) +{ + return std::log10( x ); +} +template +inline double _exp( T x ) +{ + return std::exp( x ); +} +template +inline double _exp2( T x ) +{ + return std::exp2( x ); +} +template +inline double _sqrt( T x ) +{ + return std::sqrt( x ); +} +template +inline double _erf( T x ) +{ + return std::erf( x ); +} +template +inline double _sin( T x ) +{ + return std::sin( x ); +} +template +inline double _cos( T x ) +{ + return std::cos( x ); +} +template +inline double _tan( T x ) +{ + return std::tan( x ); +} +template +inline double _asin( T x ) +{ + return std::asin( x ); +} +template +inline double _acos( T x ) +{ + return std::acos( x ); +} +template +inline double _atan( T x ) +{ + return std::atan( x ); +} +template +inline double _sinh( T x ) +{ + return std::sinh( x ); +} +template +inline double _cosh( T x ) +{ + return std::cosh( x ); +} +template +inline double _tanh( T x ) +{ + return std::tanh( x ); +} +template +inline double _asinh( T x ) +{ + return std::asinh( x ); +} +template +inline double _acosh( T x ) +{ + return std::acosh( x ); +} +template +inline double _atanh( T x ) +{ + return std::atanh( x ); +} + +inline bool _not_( bool x ) { return !x; } +inline int64_t _bitwise_not( int64_t x ) { return ~x; } + +#define EXPORT_UNARY_OP( Name, ArgType, OutType, Func ) \ + EXPORT_TEMPLATE_CPPNODE( Name, SINGLE_ARG( _unary_op ) ) +EXPORT_UNARY_OP( abs_f, double, double, abs ); +EXPORT_UNARY_OP( abs_i, int64_t, int64_t, abs ); +EXPORT_UNARY_OP( ln_f, double, double, ln ); +EXPORT_UNARY_OP( ln_i, int64_t, double, ln ); +EXPORT_UNARY_OP( log2_f, double, double, log2 ); +EXPORT_UNARY_OP( log2_i, int64_t, double, log2 ); +EXPORT_UNARY_OP( log10_f, double, double, log10 ); +EXPORT_UNARY_OP( log10_i, int64_t, double, log10 ); +EXPORT_UNARY_OP( exp_f, double, double, exp ); +EXPORT_UNARY_OP( exp_i, int64_t, double, exp ); +EXPORT_UNARY_OP( exp2_f, double, double, exp2 ); +EXPORT_UNARY_OP( exp2_i, int64_t, double, exp2 ); +EXPORT_UNARY_OP( sqrt_f, double, double, sqrt ); +EXPORT_UNARY_OP( sqrt_i, int64_t, double, sqrt ); +EXPORT_UNARY_OP( erf_f, double, double, erf ); +EXPORT_UNARY_OP( erf_i, int64_t, double, erf ); +EXPORT_UNARY_OP( sin_f, double, double, sin ); +EXPORT_UNARY_OP( sin_i, int64_t, double, sin ); +EXPORT_UNARY_OP( cos_f, double, double, cos ); +EXPORT_UNARY_OP( cos_i, int64_t, double, cos ); +EXPORT_UNARY_OP( tan_f, double, double, tan ); +EXPORT_UNARY_OP( tan_i, int64_t, double, tan ); +EXPORT_UNARY_OP( asin_f, double, double, asin ); +EXPORT_UNARY_OP( asin_i, int64_t, double, asin ); +EXPORT_UNARY_OP( acos_f, double, double, acos ); +EXPORT_UNARY_OP( acos_i, int64_t, double, acos ); +EXPORT_UNARY_OP( atan_f, double, double, atan ); +EXPORT_UNARY_OP( atan_i, int64_t, double, atan ); +EXPORT_UNARY_OP( sinh_f, double, double, sinh ); +EXPORT_UNARY_OP( sinh_i, int64_t, double, sinh ); +EXPORT_UNARY_OP( cosh_f, double, double, cosh ); +EXPORT_UNARY_OP( cosh_i, int64_t, double, cosh ); +EXPORT_UNARY_OP( tanh_f, double, double, tanh ); +EXPORT_UNARY_OP( tanh_i, int64_t, double, tanh ); +EXPORT_UNARY_OP( asinh_f, double, double, asinh ); +EXPORT_UNARY_OP( asinh_i, int64_t, double, asinh ); +EXPORT_UNARY_OP( acosh_f, double, double, acosh ); +EXPORT_UNARY_OP( acosh_i, int64_t, double, acosh ); +EXPORT_UNARY_OP( atanh_f, double, double, atanh ); +EXPORT_UNARY_OP( atanh_i, int64_t, double, atanh ); +EXPORT_UNARY_OP( not_, bool, bool, not_ ); +EXPORT_UNARY_OP( bitwise_not, int64_t, int64_t, bitwise_not ); #undef EXPORT_UNARY_OP // Binary operation -template +template DECLARE_CPPNODE( _binary_op ) { TS_INPUT( ArgT, x ); TS_INPUT( ArgT, y ); TS_OUTPUT( OutT ); - //Expanded out INIT_CPPNODE without create call... + // Expanded out INIT_CPPNODE without create call... CSP csp; const char * name() const override { return "_binary_op"; } public: _STATIC_CREATE_METHOD( SINGLE_ARG( _binary_op ) ); - _binary_op( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} + _binary_op( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) + { + } INVOKE() { @@ -126,50 +207,99 @@ DECLARE_CPPNODE( _binary_op ) }; // Math ops -template inline T _add( T x, T y ){ return x + y; } -template inline T _sub( T x, T y ){ return x - y; } -template inline T _mul( T x, T y ){ return x * y; } -template inline T _max( T x, T y ){ return std::max( x, y ); } -template inline T _min( T x, T y ){ return std::min( x, y ); } -template inline double _div( T x, T y ){ return x / ( double )y; } -inline int64_t _pow_i( int64_t x, int64_t y ){ return ( int64_t )pow( ( double )x, ( double )y ); } -inline double _pow_f( double x, double y ){ return pow( x, y ); } +template +inline T _add( T x, T y ) +{ + return x + y; +} +template +inline T _sub( T x, T y ) +{ + return x - y; +} +template +inline T _mul( T x, T y ) +{ + return x * y; +} +template +inline T _max( T x, T y ) +{ + return std::max( x, y ); +} +template +inline T _min( T x, T y ) +{ + return std::min( x, y ); +} +template +inline double _div( T x, T y ) +{ + return x / (double)y; +} +inline int64_t _pow_i( int64_t x, int64_t y ) { return (int64_t)pow( (double)x, (double)y ); } +inline double _pow_f( double x, double y ) { return pow( x, y ); } // Comparison ops -template inline bool _eq( T x, T y ){ return x == y; } -template inline bool _ne( T x, T y ){ return x != y; } -template inline bool _gt( T x, T y ){ return x > y; } -template inline bool _ge( T x, T y ){ return x >= y; } -template inline bool _lt( T x, T y ){ return x < y; } -template inline bool _le( T x, T y ){ return x <= y; } - -#define EXPORT_BINARY_OP( Name, ArgType, OutType, Func ) EXPORT_TEMPLATE_CPPNODE( Name, SINGLE_ARG( _binary_op ) ) - EXPORT_BINARY_OP( add_i, int64_t, int64_t, _add ); - EXPORT_BINARY_OP( sub_i, int64_t, int64_t, _sub ); - EXPORT_BINARY_OP( mul_i, int64_t, int64_t, _mul ); - EXPORT_BINARY_OP( div_i, int64_t, double, _div ); - EXPORT_BINARY_OP( pow_i, int64_t, int64_t, _pow_i ); - EXPORT_BINARY_OP( max_i, int64_t, int64_t, _max ); - EXPORT_BINARY_OP( min_i, int64_t, int64_t, _min ); - EXPORT_BINARY_OP( add_f, double, double, _add ); - EXPORT_BINARY_OP( sub_f, double, double, _sub ); - EXPORT_BINARY_OP( mul_f, double, double, _mul ); - EXPORT_BINARY_OP( div_f, double, double, _div ); - EXPORT_BINARY_OP( pow_f, double, double, _pow_f ); - EXPORT_BINARY_OP( max_f, double, double, _max ); - EXPORT_BINARY_OP( min_f, double, double, _min ); - EXPORT_BINARY_OP( eq_i, int64_t, bool, _eq ); - EXPORT_BINARY_OP( ne_i, int64_t, bool, _ne ); - EXPORT_BINARY_OP( gt_i, int64_t, bool, _gt ); - EXPORT_BINARY_OP( ge_i, int64_t, bool, _ge ); - EXPORT_BINARY_OP( lt_i, int64_t, bool, _lt ); - EXPORT_BINARY_OP( le_i, int64_t, bool, _le ); - EXPORT_BINARY_OP( eq_f, double, bool, _eq ); - EXPORT_BINARY_OP( ne_f, double, bool, _ne ); - EXPORT_BINARY_OP( gt_f, double, bool, _gt ); - EXPORT_BINARY_OP( ge_f, double, bool, _ge ); - EXPORT_BINARY_OP( lt_f, double, bool, _lt ); - EXPORT_BINARY_OP( le_f, double, bool, _le ); +template +inline bool _eq( T x, T y ) +{ + return x == y; +} +template +inline bool _ne( T x, T y ) +{ + return x != y; +} +template +inline bool _gt( T x, T y ) +{ + return x > y; +} +template +inline bool _ge( T x, T y ) +{ + return x >= y; +} +template +inline bool _lt( T x, T y ) +{ + return x < y; +} +template +inline bool _le( T x, T y ) +{ + return x <= y; +} + +#define EXPORT_BINARY_OP( Name, ArgType, OutType, Func ) \ + EXPORT_TEMPLATE_CPPNODE( Name, SINGLE_ARG( _binary_op ) ) +EXPORT_BINARY_OP( add_i, int64_t, int64_t, _add ); +EXPORT_BINARY_OP( sub_i, int64_t, int64_t, _sub ); +EXPORT_BINARY_OP( mul_i, int64_t, int64_t, _mul ); +EXPORT_BINARY_OP( div_i, int64_t, double, _div ); +EXPORT_BINARY_OP( pow_i, int64_t, int64_t, _pow_i ); +EXPORT_BINARY_OP( max_i, int64_t, int64_t, _max ); +EXPORT_BINARY_OP( min_i, int64_t, int64_t, _min ); +EXPORT_BINARY_OP( add_f, double, double, _add ); +EXPORT_BINARY_OP( sub_f, double, double, _sub ); +EXPORT_BINARY_OP( mul_f, double, double, _mul ); +EXPORT_BINARY_OP( div_f, double, double, _div ); +EXPORT_BINARY_OP( pow_f, double, double, _pow_f ); +EXPORT_BINARY_OP( max_f, double, double, _max ); +EXPORT_BINARY_OP( min_f, double, double, _min ); +EXPORT_BINARY_OP( eq_i, int64_t, bool, _eq ); +EXPORT_BINARY_OP( ne_i, int64_t, bool, _ne ); +EXPORT_BINARY_OP( gt_i, int64_t, bool, _gt ); +EXPORT_BINARY_OP( ge_i, int64_t, bool, _ge ); +EXPORT_BINARY_OP( lt_i, int64_t, bool, _lt ); +EXPORT_BINARY_OP( le_i, int64_t, bool, _le ); +EXPORT_BINARY_OP( eq_f, double, bool, _eq ); +EXPORT_BINARY_OP( ne_f, double, bool, _ne ); +EXPORT_BINARY_OP( gt_f, double, bool, _gt ); +EXPORT_BINARY_OP( ge_f, double, bool, _ge ); +EXPORT_BINARY_OP( lt_f, double, bool, _lt ); +EXPORT_BINARY_OP( le_f, double, bool, _le ); #undef EXPORT_BINARY_OP -} +} // namespace csp::cppnodes diff --git a/cpp/csp/cppnodes/statsimpl.cpp b/cpp/csp/cppnodes/statsimpl.cpp index ac68f335..f7070737 100644 --- a/cpp/csp/cppnodes/statsimpl.cpp +++ b/cpp/csp/cppnodes/statsimpl.cpp @@ -9,13 +9,10 @@ class _tick_window_updates : public _generic_tick_window_updates::_generic_tick_window_updates; _STATIC_CREATE_METHOD( _tick_window_updates ); - inline double createNan() - { - return std::numeric_limits::quiet_NaN(); - } + inline double createNan() { return std::numeric_limits::quiet_NaN(); } - inline void validateShape() { } - inline void checkValid() { } + inline void validateShape() {} + inline void checkValid() {} }; EXPORT_CPPNODE( _tick_window_updates ); @@ -26,13 +23,10 @@ class _time_window_updates : public _generic_time_window_updates::_generic_time_window_updates; _STATIC_CREATE_METHOD( _time_window_updates ); - inline double createNan() - { - return std::numeric_limits::quiet_NaN(); - } + inline double createNan() { return std::numeric_limits::quiet_NaN(); } - inline void validateShape() { } - inline void checkValid() { } + inline void validateShape() {} + inline void checkValid() {} }; EXPORT_CPPNODE( _time_window_updates ); @@ -43,10 +37,7 @@ class _cross_sectional_as_list : public _generic_cross_sectional, _cross_sectional_as_list>::_generic_cross_sectional; _STATIC_CREATE_METHOD( _cross_sectional_as_list ); - inline void computeCrossSectional() - { - s_window.copy_values( &unnamed_output().reserveSpace() ); - } + inline void computeCrossSectional() { s_window.copy_values( &unnamed_output().reserveSpace() ); } }; EXPORT_CPPNODE( _cross_sectional_as_list ); @@ -65,12 +56,9 @@ DECLARE_CPPNODE( _min_hit_by_tick ) TS_OUTPUT( bool ); - INIT_CPPNODE( _min_hit_by_tick ) { } + INIT_CPPNODE( _min_hit_by_tick ) {} - START() - { - csp.make_passive( x ); - } + START() { csp.make_passive( x ); } INVOKE() { @@ -97,7 +85,7 @@ DECLARE_CPPNODE( _in_sequence_check ) TS_INPUT( Generic, x ); TS_INPUT( Generic, y ); - INIT_CPPNODE( _in_sequence_check ) { } + INIT_CPPNODE( _in_sequence_check ) {} INVOKE() { @@ -121,7 +109,7 @@ DECLARE_CPPNODE( _sync_nan_f ) TS_NAMED_OUTPUT( double, x_sync ); TS_NAMED_OUTPUT( double, y_sync ); - INIT_CPPNODE( _sync_nan_f ) { } + INIT_CPPNODE( _sync_nan_f ) {} INVOKE() { @@ -159,19 +147,18 @@ DECLARE_CPPNODE( _compute ) STATE_VAR( DataValidator, s_computation ); TS_OUTPUT( double ); - //Expanded out INIT_CPPNODE without create call... + // Expanded out INIT_CPPNODE without create call... CSP csp; const char * name() const override { return "_compute"; } public: - _compute( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} - - START() + _compute( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) { - initDataValidator( s_computation ); } + START() { initDataValidator( s_computation ); } + virtual void initDataValidator( DataValidator & ) = 0; INVOKE() @@ -182,12 +169,12 @@ DECLARE_CPPNODE( _compute ) } if( csp.ticked( additions ) ) { - for( double x: additions.lastValue() ) + for( double x : additions.lastValue() ) s_computation.add( x ); } if( csp.ticked( removals ) ) { - for( double x: removals.lastValue() ) + for( double x : removals.lastValue() ) s_computation.remove( x ); } if( csp.ticked( trigger ) ) @@ -206,7 +193,7 @@ class _computeCommonArgs : public _compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na ); + validator = DataValidator( this->min_data_points, this->ignore_na ); } }; @@ -220,7 +207,7 @@ class _computeOneArg : public _compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na, this -> arg ); + validator = DataValidator( this->min_data_points, this->ignore_na, this->arg ); } }; @@ -235,7 +222,7 @@ class _computeTwoArg : public _compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na, this -> arg1, this -> arg2 ); + validator = DataValidator( this->min_data_points, this->ignore_na, this->arg1, this->arg2 ); } }; @@ -251,32 +238,32 @@ class _computeEMA : public _compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, true, alpha, this -> ignore_na, horizon, adjust ); + validator = DataValidator( this->min_data_points, true, alpha, this->ignore_na, horizon, adjust ); } }; // Export node templates -EXPORT_TEMPLATE_CPPNODE( _count, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _sum, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _kahan_sum, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _mean, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _first, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _prod, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _last, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _unique, SINGLE_ARG( _computeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _min_max, SINGLE_ARG( _computeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _var, SINGLE_ARG( _computeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _sem, SINGLE_ARG( _computeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _skew, SINGLE_ARG( _computeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _rank, SINGLE_ARG( _computeTwoArg ) ); -EXPORT_TEMPLATE_CPPNODE( _kurt, SINGLE_ARG( _computeTwoArg ) ); -EXPORT_TEMPLATE_CPPNODE( _ema_compute, _computeEMA ); -EXPORT_TEMPLATE_CPPNODE( _ema_adjusted, _computeEMA); +EXPORT_TEMPLATE_CPPNODE( _count, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _sum, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _kahan_sum, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _mean, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _first, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _prod, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _last, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _unique, SINGLE_ARG( _computeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _min_max, SINGLE_ARG( _computeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _var, SINGLE_ARG( _computeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _sem, SINGLE_ARG( _computeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _skew, SINGLE_ARG( _computeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _rank, SINGLE_ARG( _computeTwoArg ) ); +EXPORT_TEMPLATE_CPPNODE( _kurt, SINGLE_ARG( _computeTwoArg ) ); +EXPORT_TEMPLATE_CPPNODE( _ema_compute, _computeEMA ); +EXPORT_TEMPLATE_CPPNODE( _ema_adjusted, _computeEMA ); EXPORT_TEMPLATE_CPPNODE( _ema_debias_alpha, _computeEMA ); - // The following nodes are written independently from _compute -// They either have an additional input (i.e. ddof for covariance) or are implemented differently (i.e. weighted mean, which needs weight inputs) +// They either have an additional input (i.e. ddof for covariance) or are implemented differently (i.e. weighted mean, +// which needs weight inputs) /* Computation node for statistics requiring an int argument @@ -301,19 +288,18 @@ DECLARE_CPPNODE( _bivariate_compute ) TS_OUTPUT( double ); - //Expanded out INIT_CPPNODE without create call... + // Expanded out INIT_CPPNODE without create call... CSP csp; const char * name() const override { return "_bivariate_compute"; } public: - _bivariate_compute( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} - - START() + _bivariate_compute( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) { - initDataValidator( s_computation ); } + START() { initDataValidator( s_computation ); } + virtual void initDataValidator( DataValidator & ) = 0; INVOKE() @@ -333,7 +319,7 @@ DECLARE_CPPNODE( _bivariate_compute ) { const std::vector & removals_x = x_rem.lastValue(); const std::vector & removals_y = y_rem.lastValue(); - for ( size_t i = 0; i < removals_x.size(); i++ ) + for( size_t i = 0; i < removals_x.size(); i++ ) s_computation.remove( removals_x[i], removals_y[i] ); } if( csp.ticked( trigger ) ) @@ -352,7 +338,7 @@ class _bivarComputeCommonArgs : public _bivariate_compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na ); + validator = DataValidator( this->min_data_points, this->ignore_na ); } }; @@ -366,7 +352,7 @@ class _bivarComputeOneArg : public _bivariate_compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na, this -> arg ); + validator = DataValidator( this->min_data_points, this->ignore_na, this->arg ); } }; @@ -381,18 +367,17 @@ class _bivarComputeTwoArg : public _bivariate_compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na, this -> arg1, this -> arg2 ); + validator = DataValidator( this->min_data_points, this->ignore_na, this->arg1, this->arg2 ); } }; -EXPORT_TEMPLATE_CPPNODE( _weighted_mean, _bivarComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _corr, _bivarComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _weighted_var, SINGLE_ARG( _bivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _weighted_sem, SINGLE_ARG( _bivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _covar, SINGLE_ARG( _bivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _weighted_skew, SINGLE_ARG( _bivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _weighted_kurt, SINGLE_ARG( _bivarComputeTwoArg ) ); - +EXPORT_TEMPLATE_CPPNODE( _weighted_mean, _bivarComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _corr, _bivarComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _weighted_var, SINGLE_ARG( _bivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _weighted_sem, SINGLE_ARG( _bivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _covar, SINGLE_ARG( _bivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _weighted_skew, SINGLE_ARG( _bivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _weighted_kurt, SINGLE_ARG( _bivarComputeTwoArg ) ); // Trivariate and multivariate statistics template @@ -414,12 +399,9 @@ DECLARE_CPPNODE( _trivariate_compute ) TS_OUTPUT( double ); - INIT_CPPNODE( _trivariate_compute ) { } + INIT_CPPNODE( _trivariate_compute ) {} - START() - { - s_computation = DataValidator( min_data_points, ignore_na, arg ); - } + START() { s_computation = DataValidator( min_data_points, ignore_na, arg ); } INVOKE() { @@ -454,9 +436,9 @@ DECLARE_CPPNODE( _trivariate_compute ) }; EXPORT_TEMPLATE_CPPNODE( _weighted_covar, _trivariate_compute ); -EXPORT_TEMPLATE_CPPNODE( _weighted_corr, _trivariate_compute ); +EXPORT_TEMPLATE_CPPNODE( _weighted_corr, _trivariate_compute ); -DECLARE_CPPNODE ( _quantile ) +DECLARE_CPPNODE( _quantile ) { TS_INPUT( std::vector, additions ); TS_INPUT( std::vector, removals ); @@ -471,12 +453,9 @@ DECLARE_CPPNODE ( _quantile ) TS_LISTBASKET_OUTPUT( double ); - INIT_CPPNODE( _quantile ) { } + INIT_CPPNODE( _quantile ) {} - START() - { - s_qtl = DataValidator( min_data_points, ignore_na, quants.value(), interpolation_type ); - } + START() { s_qtl = DataValidator( min_data_points, ignore_na, quants.value(), interpolation_type ); } INVOKE() { @@ -486,24 +465,23 @@ DECLARE_CPPNODE ( _quantile ) } if( csp.ticked( additions ) ) { - for( double x: additions.lastValue() ) + for( double x : additions.lastValue() ) s_qtl.add( x ); } if( csp.ticked( removals ) ) { - for ( double x: removals.lastValue() ) + for( double x : removals.lastValue() ) s_qtl.remove( x ); } if( csp.ticked( trigger ) ) { - for ( size_t i = 0; i < quants.value().size(); i++ ) + for( size_t i = 0; i < quants.value().size(); i++ ) unnamed_output()[i].output( s_qtl.compute( i ) ); } } - }; -EXPORT_CPPNODE ( _quantile ); +EXPORT_CPPNODE( _quantile ); template DECLARE_CPPNODE( _exp_timewise ) @@ -518,16 +496,13 @@ DECLARE_CPPNODE( _exp_timewise ) STATE_VAR( DataValidator, s_computation ); TS_OUTPUT( double ); - INIT_CPPNODE( _exp_timewise ) { } + INIT_CPPNODE( _exp_timewise ) {} - START() - { - s_computation = DataValidator( min_data_points, true, halflife, now() ); - } + START() { s_computation = DataValidator( min_data_points, true, halflife, now() ); } INVOKE() { - if ( csp.ticked( reset ) ) + if( csp.ticked( reset ) ) { s_computation.reset(); } @@ -544,8 +519,8 @@ DECLARE_CPPNODE( _exp_timewise ) } }; -EXPORT_TEMPLATE_CPPNODE( _ema_timewise, _exp_timewise ); -EXPORT_TEMPLATE_CPPNODE( _ema_debias_halflife, _exp_timewise ); +EXPORT_TEMPLATE_CPPNODE( _ema_timewise, _exp_timewise ); +EXPORT_TEMPLATE_CPPNODE( _ema_debias_halflife, _exp_timewise ); DECLARE_CPPNODE( _arg_min_max ) { @@ -562,12 +537,9 @@ DECLARE_CPPNODE( _arg_min_max ) STATE_VAR( DataValidator, s_computation ); TS_OUTPUT( DateTime ); - INIT_CPPNODE( _arg_min_max ) { } + INIT_CPPNODE( _arg_min_max ) {} - START() - { - s_computation = DataValidator( min_data_points, ignore_na, max, recent ); - } + START() { s_computation = DataValidator( min_data_points, ignore_na, max, recent ); } INVOKE() { @@ -583,7 +555,7 @@ DECLARE_CPPNODE( _arg_min_max ) if( csp.ticked( removals ) ) { - for( double v: removals.lastValue() ) + for( double v : removals.lastValue() ) s_computation.remove( v ); } @@ -596,4 +568,4 @@ DECLARE_CPPNODE( _arg_min_max ) EXPORT_CPPNODE( _arg_min_max ); -} +} // namespace csp::cppnodes diff --git a/cpp/csp/engine/AdapterManager.cpp b/cpp/csp/engine/AdapterManager.cpp index 1f1a3999..e99067fb 100644 --- a/cpp/csp/engine/AdapterManager.cpp +++ b/cpp/csp/engine/AdapterManager.cpp @@ -4,24 +4,24 @@ namespace csp { -ManagedSimInputAdapter::ManagedSimInputAdapter( csp::Engine * engine, - const CspTypePtr & type, - AdapterManager *manager, - PushMode pushMode ) : InputAdapter( engine, type, pushMode ), - m_manager( manager ), - m_lastCycleCount( 0 ) +ManagedSimInputAdapter::ManagedSimInputAdapter( csp::Engine * engine, const CspTypePtr & type, AdapterManager * manager, + PushMode pushMode ) + : InputAdapter( engine, type, pushMode ) + , m_manager( manager ) + , m_lastCycleCount( 0 ) { } -AdapterManager::AdapterManager( csp::Engine * engine ) : m_engine( engine ), m_statusAdapter( nullptr ), m_started( false ) +AdapterManager::AdapterManager( csp::Engine * engine ) + : m_engine( engine ) + , m_statusAdapter( nullptr ) + , m_started( false ) { - if( !m_engine -> isRootEngine() ) + if( !m_engine->isRootEngine() ) CSP_THROW( NotImplemented, "AdapterManager support is not currently available in dynamic graphs" ); } -AdapterManager::~AdapterManager() -{ -} +AdapterManager::~AdapterManager() {} void AdapterManager::start( DateTime starttime, DateTime endtime ) { @@ -31,13 +31,11 @@ void AdapterManager::start( DateTime starttime, DateTime endtime ) scheduleTimerCB( starttime ); } -void AdapterManager::stop() -{ -} +void AdapterManager::stop() {} void AdapterManager::processSimTimerCB() { - DateTime next = processNextSimTimeSlice( rootEngine() -> now() ); + DateTime next = processNextSimTimeSlice( rootEngine()->now() ); if( !next.isNone() ) scheduleTimerCB( next ); } @@ -45,15 +43,15 @@ void AdapterManager::processSimTimerCB() StatusAdapter * AdapterManager::createStatusAdapter( CspTypePtr & type, PushMode pushMode ) { if( !m_statusAdapter ) - m_statusAdapter = m_engine -> createOwnedObject( type, pushMode, statusPushGroup() ); + m_statusAdapter = m_engine->createOwnedObject( type, pushMode, statusPushGroup() ); return m_statusAdapter; } -void AdapterManager::pushStatus( int64_t level, int64_t errCode, const std::string & errMsg, PushBatch *batch ) const +void AdapterManager::pushStatus( int64_t level, int64_t errCode, const std::string & errMsg, PushBatch * batch ) const { if( m_statusAdapter ) - m_statusAdapter -> pushStatus( level, errCode, errMsg, batch ); + m_statusAdapter->pushStatus( level, errCode, errMsg, batch ); } -} +} // namespace csp diff --git a/cpp/csp/engine/BasketInfo.cpp b/cpp/csp/engine/BasketInfo.cpp index f43e425e..de96d614 100644 --- a/cpp/csp/engine/BasketInfo.cpp +++ b/cpp/csp/engine/BasketInfo.cpp @@ -5,20 +5,20 @@ namespace csp { -InputBasketInfo::InputBasketInfo( RootEngine * rootEngine, - size_t size, bool isDynamic ) : m_inputs( nullptr ), - m_size( size ), - m_lastCycleCount( 0 ), - m_rootEngine( rootEngine ), - m_valid( false ), - m_isDynamic( isDynamic ) +InputBasketInfo::InputBasketInfo( RootEngine * rootEngine, size_t size, bool isDynamic ) + : m_inputs( nullptr ) + , m_size( size ) + , m_lastCycleCount( 0 ) + , m_rootEngine( rootEngine ) + , m_valid( false ) + , m_isDynamic( isDynamic ) { - //For dynamic baskets, the shape time series is stored at index -1 - //so that size / iteration still works cleanly + // For dynamic baskets, the shape time series is stored at index -1 + // so that size / iteration still works cleanly auto numElements = m_size + ( isDynamic ? 1 : 0 ); if( numElements > 0 ) { - m_inputs = ( TimeSeriesProvider const ** ) malloc( sizeof( TimeSeriesProvider * ) * numElements ); + m_inputs = (TimeSeriesProvider const **)malloc( sizeof( TimeSeriesProvider * ) * numElements ); memset( m_inputs, 0, sizeof( TimeSeriesProvider * ) * numElements ); if( isDynamic ) ++m_inputs; @@ -38,37 +38,39 @@ int64_t DynamicInputBasketInfo::addDynamicKey( const DialectGenericType & key, c int64_t elemId = m_size; ++m_size; - //we always add to the end of the basket, removes swap out last elem into removed slot + // we always add to the end of the basket, removes swap out last elem into removed slot if( elemId >= m_capacity ) { CSP_ASSERT( elemId == m_capacity ); m_capacity = std::max( 1, m_capacity * 2 ); - m_inputs = ( TimeSeriesProvider const ** ) realloc( m_inputs - 1, sizeof( TimeSeriesProvider * ) * ( m_capacity + 1 ) ); + m_inputs + = (TimeSeriesProvider const **)realloc( m_inputs - 1, sizeof( TimeSeriesProvider * ) * ( m_capacity + 1 ) ); ++m_inputs; std::fill( m_inputs + elemId, m_inputs + m_capacity, nullptr ); } else - CSP_ASSERT( !m_inputs[ elemId ] ); + CSP_ASSERT( !m_inputs[elemId] ); - m_inputs[ elemId ] = ts; - const_cast( ts ) -> setTickCountPolicy( m_tickCountPolicy ); - const_cast( ts ) -> setTickTimeWindowPolicy( m_timeWindowPolicy ); + m_inputs[elemId] = ts; + const_cast( ts )->setTickCountPolicy( m_tickCountPolicy ); + const_cast( ts )->setTickTimeWindowPolicy( m_timeWindowPolicy ); - if( likely( ( bool ) m_changeCallback ) ) + if( likely( (bool)m_changeCallback ) ) m_changeCallback( key, true, elemId, -1 ); return elemId; } -void DynamicInputBasketInfo::removeDynamicKey( uint64_t engineCycle, const DialectGenericType & key, int64_t removeId, int64_t replaceId ) +void DynamicInputBasketInfo::removeDynamicKey( uint64_t engineCycle, const DialectGenericType & key, int64_t removeId, + int64_t replaceId ) { CSP_ASSERT( m_size > 0 ); - //we need to update ticked items if replaceId is set ( being moved ) AND it ticked. Note that we dont have to reset removeId - //since we dont allow removing a key same cycle as it is ticked. - //We check if replaceId has a tick this cycle to avoid doing a potentially expensive tickeditems scan if we dont need to - if( replaceId != -1 && elem( replaceId ) -> lastCycleCount() == engineCycle ) + // we need to update ticked items if replaceId is set ( being moved ) AND it ticked. Note that we dont have to + // reset removeId since we dont allow removing a key same cycle as it is ticked. We check if replaceId has a tick + // this cycle to avoid doing a potentially expensive tickeditems scan if we dont need to + if( replaceId != -1 && elem( replaceId )->lastCycleCount() == engineCycle ) { - //need to scan unfortunately, we had a long debate back and forth on this, we may revisit in the future + // need to scan unfortunately, we had a long debate back and forth on this, we may revisit in the future for( auto & id : m_tickedInputs ) { if( id == replaceId ) @@ -81,36 +83,36 @@ void DynamicInputBasketInfo::removeDynamicKey( uint64_t engineCycle, const Diale --m_size; - m_inputs[ removeId ] = nullptr; + m_inputs[removeId] = nullptr; if( replaceId >= 0 ) { - m_inputs[ removeId ] = m_inputs[ replaceId ]; - m_inputs[ replaceId ] = nullptr; + m_inputs[removeId] = m_inputs[replaceId]; + m_inputs[replaceId] = nullptr; } - if( likely( ( bool ) m_changeCallback ) ) + if( likely( (bool)m_changeCallback ) ) m_changeCallback( key, false, removeId, replaceId ); } - -OutputBasketInfo::OutputBasketInfo( CspTypePtr & type, Node * node, - size_t size, bool isDynamic ) : m_outputs( nullptr ), - m_size( size ), - m_isDynamic( isDynamic ) +OutputBasketInfo::OutputBasketInfo( CspTypePtr & type, Node * node, size_t size, bool isDynamic ) + : m_outputs( nullptr ) + , m_size( size ) + , m_isDynamic( isDynamic ) { if( m_size > 0 ) { - //for static baskets we do a single alloc so all time series are in the same memory block + // for static baskets we do a single alloc so all time series are in the same memory block size_t allocSize = sizeof( TimeSeriesProvider * ) * m_size + sizeof( TimeSeriesProvider ) * size; - void * mem = malloc( allocSize ); - m_outputs = ( TimeSeriesProvider * * ) mem; - //location of first contiguous TSP - TimeSeriesProvider * p = reinterpret_cast( reinterpret_cast( mem ) + sizeof( TimeSeriesProvider * ) * m_size ); + void * mem = malloc( allocSize ); + m_outputs = (TimeSeriesProvider **)mem; + // location of first contiguous TSP + TimeSeriesProvider * p = reinterpret_cast( reinterpret_cast( mem ) + + sizeof( TimeSeriesProvider * ) * m_size ); for( int64_t elemId = 0; elemId < m_size; ++elemId ) { - new ( p ) TimeSeriesProvider(); - m_outputs[ elemId ] = p; - m_outputs[ elemId ] -> init( type, node ); + new( p ) TimeSeriesProvider(); + m_outputs[elemId] = p; + m_outputs[elemId]->init( type, node ); ++p; } } @@ -119,16 +121,16 @@ OutputBasketInfo::OutputBasketInfo( CspTypePtr & type, Node * node, OutputBasketInfo::~OutputBasketInfo() { for( int64_t i = 0; i < m_size; ++i ) - m_outputs[i] -> ~TimeSeriesProvider(); + m_outputs[i]->~TimeSeriesProvider(); free( m_outputs ); } - -DynamicOutputBasketInfo::DynamicOutputBasketInfo( CspTypePtr & type, Node * node ) : OutputBasketInfo( type, node, 0, true ), - m_elemType( type ), - m_parentNode( node ), - m_capacity( 0 ), - m_ownTs( true ) +DynamicOutputBasketInfo::DynamicOutputBasketInfo( CspTypePtr & type, Node * node ) + : OutputBasketInfo( type, node, 0, true ) + , m_elemType( type ) + , m_parentNode( node ) + , m_capacity( 0 ) + , m_ownTs( true ) { static CspTypePtr s_shapeType = std::make_shared( csp::autogen::DynamicBasketEvent::meta() ); m_shapeTs.init( s_shapeType, node ); @@ -136,41 +138,40 @@ DynamicOutputBasketInfo::DynamicOutputBasketInfo( CspTypePtr & type, Node * node DynamicOutputBasketInfo::~DynamicOutputBasketInfo() { - //we need to free indiviual timeseries here since we allocate TS's independently in dynamic baskets - //static baskets do a single allocation for all ts - //we may be a dynamic graph output basket, in which case we dont own the time series ( see comment in header ) + // we need to free indiviual timeseries here since we allocate TS's independently in dynamic baskets + // static baskets do a single allocation for all ts + // we may be a dynamic graph output basket, in which case we dont own the time series ( see comment in header ) if( m_ownTs ) { for( int64_t i = 0; i < m_capacity; ++i ) delete m_outputs[i]; } - //set size to 0 so that ~OutputBasketInfo doesnt try to destroy providers again + // set size to 0 so that ~OutputBasketInfo doesnt try to destroy providers again m_size = m_capacity = 0; } void DynamicOutputBasketInfo::linkInputBasket( Node * node, INOUT_ID_TYPE inputIdx ) { - node -> inputBasket( inputIdx ) -> setElem( -1, &m_shapeTs ); + node->inputBasket( inputIdx )->setElem( -1, &m_shapeTs ); m_shapeTs.addConsumer( node, InputId( inputIdx ) ); } void DynamicOutputBasketInfo::addShapeChange( const DialectGenericType & key, bool added ) { - if( m_parentNode -> rootEngine() -> cycleCount() != m_shapeTs.lastCycleCount() ) + if( m_parentNode->rootEngine()->cycleCount() != m_shapeTs.lastCycleCount() ) { auto events = autogen::DynamicBasketEvents::create(); - events -> set_events( {} ); - m_shapeTs.outputTickTyped( m_parentNode -> rootEngine() -> cycleCount(), - m_parentNode -> rootEngine() -> now(), - events, false ); + events->set_events( {} ); + m_shapeTs.outputTickTyped( m_parentNode->rootEngine()->cycleCount(), + m_parentNode->rootEngine()->now(), events, false ); } - auto & events = m_shapeTs.lastValueTyped() -> events(); + auto & events = m_shapeTs.lastValueTyped()->events(); auto event = autogen::DynamicBasketEvent::create(); - event -> set_key( key ); - event -> set_added( added ); + event->set_key( key ); + event->set_added( added ); const_cast &>( events ).emplace_back( event ); } @@ -180,10 +181,10 @@ bool DynamicOutputBasketInfo::addCapacity() if( m_size == m_capacity ) { m_capacity = std::max( 1, m_capacity * 2 ); - //in dynamic baskets we cant use the "single alloc" optimization we do for static baskets since - //input baskets already refer to the TSPs by pointer ( which can be changed on realloc ) - //so we do a regular alloc of array of ptrs, then allocate each item independently - m_outputs = ( TimeSeriesProvider ** ) realloc( m_outputs, sizeof( TimeSeriesProvider * ) * ( m_capacity ) ); + // in dynamic baskets we cant use the "single alloc" optimization we do for static baskets since + // input baskets already refer to the TSPs by pointer ( which can be changed on realloc ) + // so we do a regular alloc of array of ptrs, then allocate each item independently + m_outputs = (TimeSeriesProvider **)realloc( m_outputs, sizeof( TimeSeriesProvider * ) * ( m_capacity ) ); memset( m_outputs + m_size, 0, sizeof( TimeSeriesProvider * ) * ( m_capacity - m_size ) ); return true; } @@ -193,13 +194,14 @@ bool DynamicOutputBasketInfo::addCapacity() void DynamicOutputBasketInfo::propagateAddKey( const DialectGenericType & key, const TimeSeriesProvider * ts ) { - m_shapeTs.propagator().apply( [ts, &key]( Consumer * node, const InputId & id ) - { - static_cast( node ) -> addDynamicInputBasketKey( id.id, key, ts ); - //invoke Consumer::handleEvent not Node::handleEvent, we dont want shape tick to get into - //tickedInputs of basket - static_cast( node ) -> Consumer::handleEvent( id ); - } ); + m_shapeTs.propagator().apply( + [ts, &key]( Consumer * node, const InputId & id ) + { + static_cast( node )->addDynamicInputBasketKey( id.id, key, ts ); + // invoke Consumer::handleEvent not Node::handleEvent, we dont want shape tick to get into + // tickedInputs of basket + static_cast( node )->Consumer::handleEvent( id ); + } ); } int64_t DynamicOutputBasketInfo::addDynamicTs( const DialectGenericType & key, const TimeSeriesProvider * ts ) @@ -209,13 +211,12 @@ int64_t DynamicOutputBasketInfo::addDynamicTs( const DialectGenericType & key, c m_ownTs = false; addCapacity(); - int64_t elemId = m_size++; - m_outputs[ elemId ] = const_cast( ts ); + int64_t elemId = m_size++; + m_outputs[elemId] = const_cast( ts ); addShapeChange( key, true ); propagateAddKey( key, ts ); return elemId; - } int64_t DynamicOutputBasketInfo::addDynamicKey( const DialectGenericType & key ) @@ -227,14 +228,14 @@ int64_t DynamicOutputBasketInfo::addDynamicKey( const DialectGenericType & key ) int64_t elemId = m_size++; - if( !m_outputs[ elemId ] ) + if( !m_outputs[elemId] ) { - m_outputs[ elemId ] = new TimeSeriesProvider(); - m_outputs[ elemId ] -> init( m_elemType, m_parentNode ); + m_outputs[elemId] = new TimeSeriesProvider(); + m_outputs[elemId]->init( m_elemType, m_parentNode ); } addShapeChange( key, true ); - propagateAddKey( key, m_outputs[ elemId ] ); + propagateAddKey( key, m_outputs[elemId] ); return elemId; } @@ -243,29 +244,30 @@ int64_t DynamicOutputBasketInfo::removeDynamicKey( const DialectGenericType & ke CSP_ASSERT( m_size > 0 ); int64_t replaceId = -1; - m_outputs[ elemId ] -> reset(); + m_outputs[elemId]->reset(); if( elemId != m_size - 1 ) { - std::swap( m_outputs[ elemId ], m_outputs[ m_size - 1 ] ); + std::swap( m_outputs[elemId], m_outputs[m_size - 1] ); replaceId = m_size - 1; - //We need to update the input ids associated with this TS now that it was moved - m_outputs[ elemId ] -> propagator().apply( [elemId]( Consumer *, const InputId & id ) - { const_cast( id ).elemId = elemId; } ); + // We need to update the input ids associated with this TS now that it was moved + m_outputs[elemId]->propagator().apply( [elemId]( Consumer *, const InputId & id ) + { const_cast( id ).elemId = elemId; } ); } --m_size; addShapeChange( key, false ); - m_shapeTs.propagator().apply( [&key,elemId,replaceId]( Consumer * node, const InputId & id ) - { - static_cast( node ) -> removeDynamicInputBasketKey( id.id, key, elemId, replaceId ); - static_cast( node ) -> Consumer::handleEvent( id ); - } ); + m_shapeTs.propagator().apply( + [&key, elemId, replaceId]( Consumer * node, const InputId & id ) + { + static_cast( node )->removeDynamicInputBasketKey( id.id, key, elemId, replaceId ); + static_cast( node )->Consumer::handleEvent( id ); + } ); return replaceId; } -} +} // namespace csp diff --git a/cpp/csp/engine/Consumer.cpp b/cpp/csp/engine/Consumer.cpp index 4217bf0b..79c6584f 100644 --- a/cpp/csp/engine/Consumer.cpp +++ b/cpp/csp/engine/Consumer.cpp @@ -5,23 +5,18 @@ namespace csp { -Consumer::Consumer( Engine * engine ) : m_engine( engine ), - m_next( nullptr ), - m_rank( -1 ), - m_started( false ) +Consumer::Consumer( Engine * engine ) + : m_engine( engine ) + , m_next( nullptr ) + , m_rank( -1 ) + , m_started( false ) { } -Consumer::~Consumer() -{ -} +Consumer::~Consumer() {} -void Consumer::start() -{ -} +void Consumer::start() {} -void Consumer::stop() -{ -} +void Consumer::stop() {} -} +} // namespace csp diff --git a/cpp/csp/engine/CspEnum.cpp b/cpp/csp/engine/CspEnum.cpp index 9bed122a..40e2777a 100644 --- a/cpp/csp/engine/CspEnum.cpp +++ b/cpp/csp/engine/CspEnum.cpp @@ -3,33 +3,29 @@ namespace csp { -static CspEnumInstance s_stubInstance( "", 0, new CspEnumMeta( "", CspEnumMeta::ValueDef{{ "", 0 }} ) ); +static CspEnumInstance s_stubInstance( "", 0, new CspEnumMeta( "", CspEnumMeta::ValueDef{ { "", 0 } } ) ); -CspEnum::CspEnum() -{ - m_instance = &s_stubInstance; -} +CspEnum::CspEnum() { m_instance = &s_stubInstance; } -CspEnumMeta::CspEnumMeta( const std::string & name, const ValueDef & def ) : m_name( name ) +CspEnumMeta::CspEnumMeta( const std::string & name, const ValueDef & def ) + : m_name( name ) { - for( auto [ key,value ] : def ) + for( auto [key, value] : def ) { auto [rit, inserted] = m_instanceMap.emplace( value, std::make_shared( key, value, this ) ); if( !inserted ) CSP_THROW( TypeError, "CspEnum type " << name << " defined with multiple entries for " << value ); - m_mapping[ rit -> second -> name().c_str() ] = rit; + m_mapping[rit->second->name().c_str()] = rit; } } -CspEnumMeta::~CspEnumMeta() -{ -} +CspEnumMeta::~CspEnumMeta() {} -std::ostream &operator<<( std::ostream &os, const CspEnum & rhs ) +std::ostream & operator<<( std::ostream & os, const CspEnum & rhs ) { os << rhs.name(); return os; }; -} +} // namespace csp diff --git a/cpp/csp/engine/CspType.cpp b/cpp/csp/engine/CspType.cpp index 884fc17c..de99c4e3 100644 --- a/cpp/csp/engine/CspType.cpp +++ b/cpp/csp/engine/CspType.cpp @@ -4,43 +4,24 @@ namespace csp { -INIT_CSP_ENUM( CspType::Type, - "UNKNOWN", - "BOOL", - "INT8", - "UINT8", - "INT16", - "UINT16", - "INT32", - "UINT32", - "INT64", - "UINT64", - "DOUBLE", - "DATETIME", - "TIMEDELTA", - "DATE", - "TIME", - "ENUM", - "STRING", - "STRUCT", - "ARRAY", - "DIALECT_GENERIC" - ); +INIT_CSP_ENUM( CspType::Type, "UNKNOWN", "BOOL", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", + "UINT64", "DOUBLE", "DATETIME", "TIMEDELTA", "DATE", "TIME", "ENUM", "STRING", "STRUCT", "ARRAY", + "DIALECT_GENERIC" ); CspTypePtr & CspArrayType::create( const CspTypePtr & elemType, bool isPyStructFastList ) { using Cache = std::unordered_map; static std::mutex s_mutex; - static Cache s_cache; - static Cache s_pyStructFastListCache; + static Cache s_cache; + static Cache s_pyStructFastListCache; auto & cache = isPyStructFastList ? s_pyStructFastListCache : s_cache; std::lock_guard guard( s_mutex ); auto rv = cache.emplace( elemType.get(), nullptr ); if( rv.second ) - rv.first -> second = std::make_shared( elemType, isPyStructFastList ); - return rv.first -> second; + rv.first->second = std::make_shared( elemType, isPyStructFastList ); + return rv.first->second; } -} +} // namespace csp diff --git a/cpp/csp/engine/CycleStepTable.cpp b/cpp/csp/engine/CycleStepTable.cpp index 9805e455..2df44ab3 100644 --- a/cpp/csp/engine/CycleStepTable.cpp +++ b/cpp/csp/engine/CycleStepTable.cpp @@ -7,15 +7,15 @@ namespace csp { -static Consumer *s_END_MARKER = reinterpret_cast( 0x1 ); +static Consumer * s_END_MARKER = reinterpret_cast( 0x1 ); -CycleStepTable::CycleStepTable() : m_maxRank( -1 ), m_rankBitset() +CycleStepTable::CycleStepTable() + : m_maxRank( -1 ) + , m_rankBitset() { } -CycleStepTable::~CycleStepTable() -{ -} +CycleStepTable::~CycleStepTable() {} void CycleStepTable::resize( int32_t maxRank ) { @@ -29,12 +29,12 @@ void CycleStepTable::resize( int32_t maxRank ) void CycleStepTable::schedule( Consumer * node ) { - //already scheduled - if( node -> next() != nullptr ) + // already scheduled + if( node->next() != nullptr ) return; - int32_t rank = node -> rank(); - auto & entry = m_table[ rank ]; + int32_t rank = node->rank(); + auto & entry = m_table[rank]; if( !entry.head ) { @@ -43,48 +43,48 @@ void CycleStepTable::schedule( Consumer * node ) } else { - entry.tail -> setNext( node ); + entry.tail->setNext( node ); entry.tail = node; } - node -> setNext( s_END_MARKER ); + node->setNext( s_END_MARKER ); } void CycleStepTable::executeCycle( csp::Profiler * profiler, bool isDynamic ) { if( unlikely( profiler && !isDynamic ) ) // prioritize case without profiler - profiler -> startCycle(); + profiler->startCycle(); auto curRank = m_rankBitset.find_first(); while( curRank != DynamicBitSet<>::npos ) { m_rankBitset.reset( curRank ); - auto * curConsumer = m_table[ curRank ].head; + auto * curConsumer = m_table[curRank].head; // no real need to set tail to nullptr - m_table[ curRank ].head = nullptr; + m_table[curRank].head = nullptr; CSP_ASSERT( curConsumer != s_END_MARKER ); - + while( curConsumer != s_END_MARKER ) { - if( unlikely( ( bool )profiler ) ) + if( unlikely( (bool)profiler ) ) { - profiler -> startNode(); - curConsumer -> execute(); - profiler -> finishNode( curConsumer -> name() ); + profiler->startNode(); + curConsumer->execute(); + profiler->finishNode( curConsumer->name() ); } else - curConsumer -> execute(); + curConsumer->execute(); Consumer * prevConsumer = curConsumer; - curConsumer = curConsumer -> next(); - prevConsumer -> setNext( nullptr ); + curConsumer = curConsumer->next(); + prevConsumer->setNext( nullptr ); } curRank = m_rankBitset.find_next( curRank ); } if( unlikely( profiler && !isDynamic ) ) - profiler -> finishCycle(); + profiler->finishCycle(); } -} +} // namespace csp diff --git a/cpp/csp/engine/Dictionary.cpp b/cpp/csp/engine/Dictionary.cpp index d9e2fba6..e4bafc84 100644 --- a/cpp/csp/engine/Dictionary.cpp +++ b/cpp/csp/engine/Dictionary.cpp @@ -1,12 +1,13 @@ #include #include -template<> struct std::hash> +template<> +struct std::hash> { size_t operator()( const vector & v ) { size_t h = 0; - for( auto &&entry : v ) + for( auto && entry : v ) h ^= hash()( entry._data ); return h; } @@ -15,11 +16,9 @@ template<> struct std::hash> namespace csp { -Dictionary::Dictionary() -{} +Dictionary::Dictionary() {} -Dictionary::~Dictionary() -{} +Dictionary::~Dictionary() {} Dictionary::Dictionary( const Dictionary & rhs ) { @@ -30,7 +29,7 @@ Dictionary::Dictionary( const Dictionary & rhs ) Dictionary::Dictionary( Dictionary && rhs ) { m_map = std::move( rhs.m_map ); - m_data = std::move( rhs.m_data); + m_data = std::move( rhs.m_data ); } Dictionary & Dictionary::operator=( const Dictionary & rhs ) @@ -43,29 +42,26 @@ Dictionary & Dictionary::operator=( const Dictionary & rhs ) Dictionary & Dictionary::operator=( Dictionary && rhs ) { m_map = std::move( rhs.m_map ); - m_data = std::move( rhs.m_data); + m_data = std::move( rhs.m_data ); return *this; } -bool Dictionary::exists( const std::string & key ) const -{ - return m_map.find( key ) != m_map.end(); -} +bool Dictionary::exists( const std::string & key ) const { return m_map.find( key ) != m_map.end(); } bool Dictionary::operator==( const Dictionary & rhs ) const { - if( m_data.size() != rhs.m_data.size()) + if( m_data.size() != rhs.m_data.size() ) { return false; } - for(auto&& entry:m_data) + for( auto && entry : m_data ) { - auto &&rhsIt = rhs.m_map.find( entry.first ); - if( rhsIt == rhs.m_map.end()) + auto && rhsIt = rhs.m_map.find( entry.first ); + if( rhsIt == rhs.m_map.end() ) { return false; } - if( entry.second != rhs.m_data[rhsIt->second].second) + if( entry.second != rhs.m_data[rhsIt->second].second ) { return false; } @@ -76,11 +72,11 @@ bool Dictionary::operator==( const Dictionary & rhs ) const size_t Dictionary::hash() const { size_t hash = 0; - for( auto &&entry: m_data ) + for( auto && entry : m_data ) { hash = hash ^ std::hash()( entry.first ) ^ std::hash()( entry.second._data ); } return hash; } -} +} // namespace csp diff --git a/cpp/csp/engine/DynamicEngine.cpp b/cpp/csp/engine/DynamicEngine.cpp index 393592ab..55321b20 100644 --- a/cpp/csp/engine/DynamicEngine.cpp +++ b/cpp/csp/engine/DynamicEngine.cpp @@ -3,40 +3,29 @@ namespace csp { -DynamicEngine::GraphOutput::GraphOutput( Engine * engine ) : OutputAdapter( engine ) +DynamicEngine::GraphOutput::GraphOutput( Engine * engine ) + : OutputAdapter( engine ) { } -DynamicEngine::GraphOutput::~GraphOutput() -{ -} - -void DynamicEngine::GraphOutput::start() -{ - input() -> removeConsumer( this, InputId( 0 ) ); -} +DynamicEngine::GraphOutput::~GraphOutput() {} +void DynamicEngine::GraphOutput::start() { input()->removeConsumer( this, InputId( 0 ) ); } -DynamicEngine::DynamicEngine( CycleStepTable & stepTable, RootEngine * root, - ShutdownFn && shutdownFn ) : Engine( stepTable, root ), - m_shutdownFn( std::move( shutdownFn ) ) +DynamicEngine::DynamicEngine( CycleStepTable & stepTable, RootEngine * root, ShutdownFn && shutdownFn ) + : Engine( stepTable, root ) + , m_shutdownFn( std::move( shutdownFn ) ) { } void DynamicEngine::registerGraphOutput( const std::string & key, GraphOutput * output ) { - //no need to check for clash, output names are gauranteed unique + // no need to check for clash, output names are gauranteed unique m_graphOutputs[key] = output; } -TimeSeriesProvider * DynamicEngine::outputTs( const std::string & key ) -{ - return m_graphOutputs[key] -> input(); -} +TimeSeriesProvider * DynamicEngine::outputTs( const std::string & key ) { return m_graphOutputs[key]->input(); } -void DynamicEngine::shutdown() -{ - m_shutdownFn(); -} +void DynamicEngine::shutdown() { m_shutdownFn(); } -} +} // namespace csp diff --git a/cpp/csp/engine/DynamicNode.cpp b/cpp/csp/engine/DynamicNode.cpp index c6feddc0..bb588975 100644 --- a/cpp/csp/engine/DynamicNode.cpp +++ b/cpp/csp/engine/DynamicNode.cpp @@ -1,51 +1,46 @@ -#include -#include #include +#include #include #include +#include namespace csp { -DynamicNode::DynamicNode( Engine * engine, const std::string & name, - std::vector snapIds, - EngineBuilder builder, - NodeDef def ) : - Node( def, engine ), - m_engineBuilder( builder ), - m_name( name ), - m_snapIds( std::move( snapIds ) ) +DynamicNode::DynamicNode( Engine * engine, const std::string & name, std::vector snapIds, + EngineBuilder builder, NodeDef def ) + : Node( def, engine ) + , m_engineBuilder( builder ) + , m_name( name ) + , m_snapIds( std::move( snapIds ) ) { - //input 0 is always the dynamic basket trigger, we can create that here + // input 0 is always the dynamic basket trigger, we can create that here initInputBasket( 0, 0, true ); } -DynamicNode::~DynamicNode() -{ - m_instanceMap.clear(); -} +DynamicNode::~DynamicNode() { m_instanceMap.clear(); } void DynamicNode::start() { - //no need to keep csp.snap inputs active + // no need to keep csp.snap inputs active for( auto id : m_snapIds ) makePassive( InputId( id ) ); - //dont need this anymore + // dont need this anymore m_snapIds.clear(); } void DynamicNode::stop() { for( auto & instance : m_instanceMap ) - instance.second -> stop(); + instance.second->stop(); } void DynamicNode::handleEvent( InputId id ) { - //see comment in addDynamicInstance... these ids can be stub ids, we dont rely on them, skip Node::handleEvent logic - //we do need to let dynamic basket input flow through the regular channels though - if( id.id == ( INOUT_ID_TYPE ) -1 ) + // see comment in addDynamicInstance... these ids can be stub ids, we dont rely on them, skip Node::handleEvent + // logic we do need to let dynamic basket input flow through the regular channels though + if( id.id == (INOUT_ID_TYPE)-1 ) Consumer::handleEvent( id ); else Node::handleEvent( id ); @@ -54,41 +49,44 @@ void DynamicNode::handleEvent( InputId id ) void DynamicNode::executeImpl() { DynamicInputBasketInfo * dynBasket = static_cast( inputBasket( 0 ) ); - auto * shapeTs = dynBasket -> shapeTs(); + auto * shapeTs = dynBasket->shapeTs(); - bool shapeTicked = shapeTs -> lastCycleCount() == rootEngine() -> cycleCount(); + bool shapeTicked = shapeTs->lastCycleCount() == rootEngine()->cycleCount(); if( shapeTicked ) { - auto & events = shapeTs -> lastValueTyped() -> events(); + auto & events = shapeTs->lastValueTyped()->events(); for( auto & event : events ) { - if( event -> added() ) - addDynamicInstance( event -> key() ); - //note that we handle removes AFTER the end of the dynamic cycle, the reason is two-fold - //1) there can be cases where remove and some tick happen in the same cycle. to be consistent with runing on endtime, we process that tick - //2) its possible some nodes in the to-be-removed sub engine are already scheduled in the cycle step table. it would be difficult to get them removed - - //HOWEVER there is the pitfall that a csp.attach() input will be accessed on the same cycle as it being shutdown. It will no longer be a valid ts in this last cycle - //since it was already removed. In python code it still "works" because the last value is cached in a localframe var. in c++ nodes access will raise since buffer will be reset to empty. - //In BOTH cases, a proper check of csp.valid() will safely return false on the removed edge + if( event->added() ) + addDynamicInstance( event->key() ); + // note that we handle removes AFTER the end of the dynamic cycle, the reason is two-fold + // 1) there can be cases where remove and some tick happen in the same cycle. to be consistent with runing + // on endtime, we process that tick 2) its possible some nodes in the to-be-removed sub engine are already + // scheduled in the cycle step table. it would be difficult to get them removed + + // HOWEVER there is the pitfall that a csp.attach() input will be accessed on the same cycle as it being + // shutdown. It will no longer be a valid ts in this last cycle since it was already removed. In python + // code it still "works" because the last value is cached in a localframe var. in c++ nodes access will + // raise since buffer will be reset to empty. In BOTH cases, a proper check of csp.valid() will safely + // return false on the removed edge } } - //execute the cycle, all inputs should have ticked in by now - m_cycleStepTable.executeCycle( rootEngine() -> profiler(), true ); + // execute the cycle, all inputs should have ticked in by now + m_cycleStepTable.executeCycle( rootEngine()->profiler(), true ); if( shapeTicked ) { - auto & events = shapeTs -> lastValueTyped() -> events(); + auto & events = shapeTs->lastValueTyped()->events(); for( auto & event : events ) { - if( !event -> added() ) - removeDynamicInstance( event -> key() ); + if( !event->added() ) + removeDynamicInstance( event->key() ); } } - //We defer csp.stop_engine shutdowns until after the cycle because we cant shutdown / destroy - //the dynamic engine while the node within the dynamic which is calling csp.stop_engine is being executed! + // We defer csp.stop_engine shutdowns until after the cycle because we cant shutdown / destroy + // the dynamic engine while the node within the dynamic which is calling csp.stop_engine is being executed! if( unlikely( !m_dynamicShutdowns.empty() ) ) { for( auto & key : m_dynamicShutdowns ) @@ -100,122 +98,124 @@ void DynamicNode::executeImpl() int64_t DynamicNode::elemId( const DialectGenericType & key ) { auto it = m_dynamicKeyMap.find( key ); - return it == m_dynamicKeyMap.end() ? InputId::ELEM_ID_NONE : it -> second; + return it == m_dynamicKeyMap.end() ? InputId::ELEM_ID_NONE : it->second; } void DynamicNode::addDynamicInstance( const DialectGenericType & key ) { - //TODO remove all the places keeping track if key mappings, move into dynamic basket impl + // TODO remove all the places keeping track if key mappings, move into dynamic basket impl int64_t keyElemId = m_dynamicElemIdMap.size(); - m_dynamicKeyMap[ key ] = keyElemId; + m_dynamicKeyMap[key] = keyElemId; m_dynamicElemIdMap.emplace_back( key ); - auto shutdownFn = [this,key]() { m_dynamicShutdowns.insert( key ); }; + auto shutdownFn = [this, key]() { m_dynamicShutdowns.insert( key ); }; auto instanceEngine = std::make_unique( m_cycleStepTable, rootEngine(), shutdownFn ); - //build it up + // build it up auto outputs = m_engineBuilder( this, instanceEngine.get(), key ); - //Wire up all outputs as new dynamic output keys + // Wire up all outputs as new dynamic output keys assert( outputs.size() == numOutputs() ); - for( INOUT_ID_TYPE idx = 0; idx < ( INOUT_ID_TYPE ) outputs.size(); ++idx ) + for( INOUT_ID_TYPE idx = 0; idx < (INOUT_ID_TYPE)outputs.size(); ++idx ) { auto * dynBasket = static_cast( outputBasket( idx ) ); - dynBasket -> addDynamicTs( key, outputs[idx] ); + dynBasket->addDynamicTs( key, outputs[idx] ); } { - //monitor will keep track of all events scheduled for "now" which we want to execute immediately after start - //otherwise they will defer to the next engine cycle. The class destructore will then process those events - Scheduler::DynamicEngineStartMonitor monitor( rootEngine() -> dynamicEngineStartMonitor() ); - instanceEngine -> start(); + // monitor will keep track of all events scheduled for "now" which we want to execute immediately after start + // otherwise they will defer to the next engine cycle. The class destructore will then process those events + Scheduler::DynamicEngineStartMonitor monitor( rootEngine()->dynamicEngineStartMonitor() ); + instanceEngine->start(); } - //Note that all external inputs are wired by graph building as we add them all as inputs to the dynamic node up front. - //However, there may be external inputs that ticked the same cycle as creating this engine. In such a case we need to - //force-propagate the input on startup to all nodes in this engine so that the new graph will see it ticked properly on the first cycle + // Note that all external inputs are wired by graph building as we add them all as inputs to the dynamic node up + // front. However, there may be external inputs that ticked the same cycle as creating this engine. In such a case + // we need to force-propagate the input on startup to all nodes in this engine so that the new graph will see it + // ticked properly on the first cycle for( auto input = inputs(); input; ++input ) { if( inputTicked( input.inputId() ) ) { - input -> propagator().apply( - [e=instanceEngine.get()]( Consumer * consumer, const InputId & inputId ) + input->propagator().apply( + [e = instanceEngine.get()]( Consumer * consumer, const InputId & inputId ) { - if( consumer -> engine() == e ) - consumer -> handleEvent( inputId ); + if( consumer->engine() == e ) + consumer->handleEvent( inputId ); } ); } } - - //Need to attach all inputs created in the dynamic instance as inputs to this outter DynamicNode - //as those inputs tick, we need to get scheduled in the root engine so that we execute the cycle step in here - //we provide a stub id that is never intended to be used. DynamicNode::handleEvent is overriden to ensure Node::handleEvent doesnt - //act on these ids + // Need to attach all inputs created in the dynamic instance as inputs to this outter DynamicNode + // as those inputs tick, we need to get scheduled in the root engine so that we execute the cycle step in here + // we provide a stub id that is never intended to be used. DynamicNode::handleEvent is overriden to ensure + // Node::handleEvent doesnt act on these ids constexpr INOUT_ID_TYPE stubId = -1; - for( auto & input : instanceEngine -> inputAdapters() ) - const_cast( input.get() ) -> addConsumer( this, InputId{ stubId } ); + for( auto & input : instanceEngine->inputAdapters() ) + const_cast( input.get() )->addConsumer( this, InputId{ stubId } ); - m_instanceMap[ key ] = std::move( instanceEngine ); + m_instanceMap[key] = std::move( instanceEngine ); } void DynamicNode::removeDynamicInstance( const DialectGenericType & key ) { auto it = m_instanceMap.find( key ); - //This can happen if a dynamic shuts itself down before the trigger basket removes the key + // This can happen if a dynamic shuts itself down before the trigger basket removes the key if( it == m_instanceMap.end() ) return; - it -> second -> stop(); + it->second->stop(); - //we need to remove consumers of anything coming in from the outter engine - //to optimize expensive removeConsumer calls we search for all inputs that are sourced externally + // we need to remove consumers of anything coming in from the outter engine + // to optimize expensive removeConsumer calls we search for all inputs that are sourced externally std::unordered_set externalTs; for( auto input = inputs(); input; ++input ) externalTs.insert( input.get() ); - for( auto & node : it -> second -> nodes() ) + for( auto & node : it->second->nodes() ) { - for( auto input_iter = node -> inputs(); input_iter; ++input_iter ) + for( auto input_iter = node->inputs(); input_iter; ++input_iter ) { if( externalTs.find( input_iter.get() ) != externalTs.end() ) - const_cast( input_iter.get() ) -> removeConsumer( node.get(), input_iter.inputId() ); + const_cast( input_iter.get() ) + ->removeConsumer( node.get(), input_iter.inputId() ); } } - for( auto & adapter : it -> second -> outputAdapters() ) + for( auto & adapter : it->second->outputAdapters() ) { - for( auto input_iter = adapter -> inputs(); input_iter; ++input_iter ) + for( auto input_iter = adapter->inputs(); input_iter; ++input_iter ) { if( externalTs.find( input_iter.get() ) != externalTs.end() ) - const_cast( input_iter.get() ) -> removeConsumer( adapter.get(), input_iter.inputId() ); + const_cast( input_iter.get() ) + ->removeConsumer( adapter.get(), input_iter.inputId() ); } } - auto removeIt = m_dynamicKeyMap.find( key ); - int64_t removeId = removeIt -> second; + auto removeIt = m_dynamicKeyMap.find( key ); + int64_t removeId = removeIt->second; - if( ( size_t) removeId != ( m_dynamicKeyMap.size() - 1 ) ) + if( (size_t)removeId != ( m_dynamicKeyMap.size() - 1 ) ) { - auto replaceId = m_dynamicKeyMap.size() - 1; - auto replacedKey = m_dynamicElemIdMap[ replaceId ]; - m_dynamicKeyMap[ replacedKey ] = removeId; - m_dynamicElemIdMap[ removeId ] = replacedKey; + auto replaceId = m_dynamicKeyMap.size() - 1; + auto replacedKey = m_dynamicElemIdMap[replaceId]; + m_dynamicKeyMap[replacedKey] = removeId; + m_dynamicElemIdMap[removeId] = replacedKey; } for( INOUT_ID_TYPE idx = 0; idx < numOutputs(); ++idx ) { auto * dynBasket = static_cast( outputBasket( idx ) ); - dynBasket -> removeDynamicKey( key, removeId ); + dynBasket->removeDynamicKey( key, removeId ); } m_dynamicKeyMap.erase( removeIt ); m_dynamicElemIdMap.pop_back(); - //destroy Engine + // destroy Engine m_instanceMap.erase( it ); } -} +} // namespace csp diff --git a/cpp/csp/engine/Engine.cpp b/cpp/csp/engine/Engine.cpp index efb14f2c..6a0e9e74 100644 --- a/cpp/csp/engine/Engine.cpp +++ b/cpp/csp/engine/Engine.cpp @@ -1,35 +1,33 @@ #include #include +#include #include #include -#include #include #include #include -#include #include +#include namespace csp { -Engine::Engine( CycleStepTable & stepTable, RootEngine * root ) : - m_rootEngine( root ? root : static_cast( this ) ), - m_cycleStepTable( stepTable ) +Engine::Engine( CycleStepTable & stepTable, RootEngine * root ) + : m_rootEngine( root ? root : static_cast( this ) ) + , m_cycleStepTable( stepTable ) { } -Engine::~Engine() -{ -} +Engine::~Engine() {} void Engine::registerOwnedObject( std::shared_ptr manager ) { - //AdapterManagers may get created in a dynamic graph the first time around, but we want them shared across dynamics - //so they always get registered and effectively owned by root engine + // AdapterManagers may get created in a dynamic graph the first time around, but we want them shared across dynamics + // so they always get registered and effectively owned by root engine if( !isRootEngine() ) - rootEngine() -> registerOwnedObject( manager ); + rootEngine()->registerOwnedObject( manager ); - m_adapterManagers.emplace_back( manager ); + m_adapterManagers.emplace_back( manager ); } void Engine::registerOwnedObject( std::unique_ptr adapter ) @@ -42,10 +40,7 @@ void Engine::registerOwnedObject( std::unique_ptr adapter ) m_outputAdapters.emplace_back( std::move( adapter ) ); } -void Engine::registerOwnedObject( std::unique_ptr node ) -{ - m_nodes.emplace_back( std::move( node ) ); -} +void Engine::registerOwnedObject( std::unique_ptr node ) { m_nodes.emplace_back( std::move( node ) ); } void Engine::registerGraphOutput( const DialectGenericType & key, std::shared_ptr adapter ) { @@ -55,25 +50,25 @@ void Engine::registerGraphOutput( const DialectGenericType & key, std::shared_pt m_graphOutputKeys.emplace_back( key ); - //If were in a dynamic engine we register the key to catch duplicates up front. We also register in root - //so that it gets processed at the end of the run. + // If were in a dynamic engine we register the key to catch duplicates up front. We also register in root + // so that it gets processed at the end of the run. if( !isRootEngine() ) - rootEngine() -> registerGraphOutput( key, adapter ); + rootEngine()->registerGraphOutput( key, adapter ); } csp::GraphOutputAdapter * Engine::graphOutput( const DialectGenericType & key ) { auto it = m_graphOutputs.find( key ); - return it == m_graphOutputs.end() ? nullptr : it -> second.get(); + return it == m_graphOutputs.end() ? nullptr : it->second.get(); } -static int32_t calcRank( Consumer * consumer, std::unordered_set & visited, std::vector & path ) +static int32_t calcRank( Consumer * consumer, std::unordered_set & visited, std::vector & path ) { - if( consumer -> rank() >= 0 ) - return consumer -> rank(); + if( consumer->rank() >= 0 ) + return consumer->rank(); - //we dont have to check visited vs done in our cycle check since done is checked above by seeing if rank was - //already computed. So we only insert into visited + // we dont have to check visited vs done in our cycle check since done is checked above by seeing if rank was + // already computed. So we only insert into visited auto rv = visited.insert( consumer ); if( !rv.second ) { @@ -81,44 +76,45 @@ static int32_t calcRank( Consumer * consumer, std::unordered_set & v oss << "Illegal cycle found in graph, path:\n\t** "; for( size_t idx = path.size() - 1; idx > 0; --idx ) { - //flag the end of the cycle clearly in the output, which is when we first see the current node again ( unless its the first iteration ) + // flag the end of the cycle clearly in the output, which is when we first see the current node again ( + // unless its the first iteration ) bool end_of_cycle = ( path[idx] == consumer ) && ( idx != path.size() - 1 ); - oss << path[idx] -> name() << ( end_of_cycle ? " ** " : "" ) << " -> "; + oss << path[idx]->name() << ( end_of_cycle ? " ** " : "" ) << " -> "; } - oss << path[0] -> name(); + oss << path[0]->name(); CSP_THROW( RuntimeException, oss.str() ); } int rank = 0; - for( auto input_iter = consumer -> inputs(); input_iter; ++input_iter ) + for( auto input_iter = consumer->inputs(); input_iter; ++input_iter ) { int input_rank = 0; - if( input_iter -> node() && - //its possible that we are a dynamic engine and the input is wired from the parent engine - //in this case we treat it as if its a 0-ranked input - input_iter -> node() -> engine() == consumer -> engine() ) + if( input_iter->node() && + // its possible that we are a dynamic engine and the input is wired from the parent engine + // in this case we treat it as if its a 0-ranked input + input_iter->node()->engine() == consumer->engine() ) { - path.push_back( input_iter -> node() ); - input_rank = calcRank( input_iter -> node(), visited, path ); + path.push_back( input_iter->node() ); + input_rank = calcRank( input_iter->node(), visited, path ); path.pop_back(); } rank = std::max( rank, input_rank + 1 ); } - consumer -> setRank( rank ); + consumer->setRank( rank ); return rank; } int32_t Engine::computeRanks() { - //We will start with output nodes and work our way back to avoid recomputing ranks repeatedly if - //we want input -> out - std::vector roots; + // We will start with output nodes and work our way back to avoid recomputing ranks repeatedly if + // we want input -> out + std::vector roots; for( auto & node : m_nodes ) { - if( node -> numOutputs() == 0 ) + if( node->numOutputs() == 0 ) roots.emplace_back( node.get() ); } @@ -127,19 +123,19 @@ int32_t Engine::computeRanks() for( auto & entry : m_graphOutputs ) { - //might be a dynamic engine graph output registered in root - if( entry.second -> engine() == this ) + // might be a dynamic engine graph output registered in root + if( entry.second->engine() == this ) roots.emplace_back( entry.second.get() ); } - - //these are used for cycle detection, which can happen with delayed bindings + + // these are used for cycle detection, which can happen with delayed bindings std::vector path; std::unordered_set visited; int32_t maxRank = 0; for( auto * consumer : roots ) { - //we dont include outputs in the path since its confusing to see in the cycle ( and clearly not part of it ) + // we dont include outputs in the path since its confusing to see in the cycle ( and clearly not part of it ) path.push_back( consumer ); maxRank = std::max( maxRank, calcRank( consumer, visited, path ) ); path.pop_back(); @@ -153,50 +149,51 @@ void Engine::start() int32_t maxRank = computeRanks(); m_cycleStepTable.resize( maxRank ); - auto start = std::max( m_rootEngine -> now(), m_rootEngine -> startTime() ); - auto end = m_rootEngine -> endTime(); + auto start = std::max( m_rootEngine->now(), m_rootEngine->startTime() ); + auto end = m_rootEngine->endTime(); - //start up managers + // start up managers for( auto & manager : m_adapterManagers ) { - manager -> start( start, end ); - manager -> setStarted(); + manager->start( start, end ); + manager->setStarted(); } - //start up output adapters + // start up output adapters for( auto & adapter : m_outputAdapters ) { - adapter -> start(); - adapter -> setStarted(); + adapter->start(); + adapter->setStarted(); } for( auto & entry : m_graphOutputs ) { auto & graphOutputAdapter = entry.second; - if( graphOutputAdapter -> engine() == this ) + if( graphOutputAdapter->engine() == this ) { - graphOutputAdapter -> start(); - graphOutputAdapter -> setStarted(); + graphOutputAdapter->start(); + graphOutputAdapter->setStarted(); } } - //start up input adapters + // start up input adapters for( auto & adapter : m_inputAdapters ) { - adapter -> start( start, end ); - adapter -> setStarted(); + adapter->start( start, end ); + adapter->setStarted(); } - //see registerOwnedObject( AdapterManager ) above, we register adapter managers with root. At this point we dont - //need the list of mgrs created in a dynamic engine anymore, so we clear out the mem ( and effetively take them out of the stop() list for dynamic shutdown ) + // see registerOwnedObject( AdapterManager ) above, we register adapter managers with root. At this point we dont + // need the list of mgrs created in a dynamic engine anymore, so we clear out the mem ( and effetively take them out + // of the stop() list for dynamic shutdown ) if( !isRootEngine() ) m_adapterManagers.clear(); - //startup nodes + // startup nodes for( auto & node : m_nodes ) { - node -> start(); - node -> setStarted(); + node->start(); + node->setStarted(); } } @@ -205,34 +202,34 @@ void Engine::stop() // Ensure we only stop nodes/adapters that have started in the case an exception occurs during startup for( auto & node : m_nodes ) { - if( node -> started() ) - node -> stop(); + if( node->started() ) + node->stop(); } for( auto & adapter : m_inputAdapters ) { - if( adapter -> started() ) - adapter -> stop(); + if( adapter->started() ) + adapter->stop(); } for( auto & entry : m_graphOutputs ) { auto & graphOutputAdapter = entry.second; - if( graphOutputAdapter -> started() && graphOutputAdapter -> engine() == this ) - graphOutputAdapter -> stop(); + if( graphOutputAdapter->started() && graphOutputAdapter->engine() == this ) + graphOutputAdapter->stop(); } for( auto & adapter : m_outputAdapters ) { - if( adapter -> started() ) - adapter -> stop(); + if( adapter->started() ) + adapter->stop(); } for( auto & manager : m_adapterManagers ) { - if( manager -> started() ) - manager -> stop(); + if( manager->started() ) + manager->stop(); } } -} +} // namespace csp diff --git a/cpp/csp/engine/Enums.cpp b/cpp/csp/engine/Enums.cpp index bd7f28a6..cf5951d4 100644 --- a/cpp/csp/engine/Enums.cpp +++ b/cpp/csp/engine/Enums.cpp @@ -3,10 +3,5 @@ namespace csp { -INIT_CSP_ENUM( PushMode, - "UNKNOWN", - "LAST_VALUE", - "NON_COLLAPSING", - "BURST" - ); +INIT_CSP_ENUM( PushMode, "UNKNOWN", "LAST_VALUE", "NON_COLLAPSING", "BURST" ); } diff --git a/cpp/csp/engine/EventPropagator.cpp b/cpp/csp/engine/EventPropagator.cpp index c5e85e3f..0640e362 100644 --- a/cpp/csp/engine/EventPropagator.cpp +++ b/cpp/csp/engine/EventPropagator.cpp @@ -4,35 +4,29 @@ namespace csp { -EventPropagator::EventPropagator() -{ -} +EventPropagator::EventPropagator() {} void EventPropagator::propagate() { - m_consumers.apply( []( Consumer * consumer, InputId id ) { consumer -> handleEvent( id ); } ); + m_consumers.apply( []( Consumer * consumer, InputId id ) { consumer->handleEvent( id ); } ); } - -//ConsumerVector +// ConsumerVector EventPropagator::ConsumerVector::ConsumerVector() { - //if this is getting constructed that means we have > 1 consumer, so start with initialization to size 2 - m_capacity = 2; - m_size = 0; - m_consumers = ( ConsumerInfo * ) malloc( sizeof( ConsumerInfo ) * m_capacity ); - m_consumers = ( ConsumerInfo * ) ( uint64_t( m_consumers ) | 0x1 ); + // if this is getting constructed that means we have > 1 consumer, so start with initialization to size 2 + m_capacity = 2; + m_size = 0; + m_consumers = (ConsumerInfo *)malloc( sizeof( ConsumerInfo ) * m_capacity ); + m_consumers = (ConsumerInfo *)( uint64_t( m_consumers ) | 0x1 ); } -EventPropagator::ConsumerVector::~ConsumerVector() -{ - free( consumers() ); -} +EventPropagator::ConsumerVector::~ConsumerVector() { free( consumers() ); } EventPropagator::ConsumerInfo * EventPropagator::ConsumerVector::findConsumer( Consumer * consumer, InputId id ) { ConsumerInfo * needle = consumers(); - ConsumerInfo * end = needle + m_size; + ConsumerInfo * end = needle + m_size; ConsumerInfo match{ consumer, id }; while( needle < end ) @@ -49,8 +43,8 @@ void EventPropagator::ConsumerVector::push_back( Consumer * consumer, InputId id if( m_size == m_capacity ) { m_capacity *= 2; - m_consumers = ( ConsumerInfo * ) realloc( consumers(), sizeof( ConsumerInfo ) * m_capacity ); - m_consumers = ( ConsumerInfo * ) ( uint64_t( m_consumers ) | 0x1 ); + m_consumers = (ConsumerInfo *)realloc( consumers(), sizeof( ConsumerInfo ) * m_capacity ); + m_consumers = (ConsumerInfo *)( uint64_t( m_consumers ) | 0x1 ); } consumers()[m_size++] = ConsumerInfo( consumer, id ); @@ -66,12 +60,12 @@ bool EventPropagator::ConsumerVector::addConsumer( Consumer * consumer, InputId bool EventPropagator::ConsumerVector::removeConsumer( Consumer * consumer, InputId id ) { - //called by make_active, same as addConsumer but ensures its not already in the list + // called by make_active, same as addConsumer but ensures its not already in the list ConsumerInfo * ci = findConsumer( consumer, id ); if( ci ) { - //avoid deleting mid-array, swap with last elem instead - *ci = consumers()[ m_size - 1 ]; + // avoid deleting mid-array, swap with last elem instead + *ci = consumers()[m_size - 1]; --m_size; return true; } @@ -79,11 +73,8 @@ bool EventPropagator::ConsumerVector::removeConsumer( Consumer * consumer, Input return false; } -//Consumers union -EventPropagator::Consumers::Consumers() -{ - m_consumerData.flag = EMPTY; -} +// Consumers union +EventPropagator::Consumers::Consumers() { m_consumerData.flag = EMPTY; } EventPropagator::Consumers::~Consumers() { @@ -102,10 +93,10 @@ bool EventPropagator::Consumers::addConsumer( Consumer * consumer, InputId id, b if( single() == ConsumerInfo{ consumer, id } ) return false; - //Copy out the single info before the unionized vector in place new destroys it! + // Copy out the single info before the unionized vector in place new destroys it! ConsumerInfo singleCI{ single() }; - //transform from single to vector + // transform from single to vector new( &consumersVector() ) ConsumerVector(); consumersVector().addConsumer( singleCI.consumer, singleCI.inputId, false ); consumersVector().addConsumer( consumer, id, false ); @@ -137,8 +128,8 @@ bool EventPropagator::Consumers::removeConsumer( Consumer * consumer, InputId id { if( consumersVector().size() == 1 ) { - //Going from vector back to single - //Copy over data before we whack the unionized memory space + // Going from vector back to single + // Copy over data before we whack the unionized memory space auto singleCI = consumersVector()[0]; consumersVector().ConsumerVector::~ConsumerVector(); single() = singleCI; @@ -160,4 +151,4 @@ void EventPropagator::Consumers::clear() } } -} +} // namespace csp diff --git a/cpp/csp/engine/GraphOutputAdapter.cpp b/cpp/csp/engine/GraphOutputAdapter.cpp index 83ae3daf..97da6cfe 100644 --- a/cpp/csp/engine/GraphOutputAdapter.cpp +++ b/cpp/csp/engine/GraphOutputAdapter.cpp @@ -4,48 +4,44 @@ namespace csp { -GraphOutputAdapter::GraphOutputAdapter( csp::Engine * engine, int32_t tickCount, TimeDelta tickHistory ) : - OutputAdapter( engine ), - m_tickHistory( tickHistory ), - m_tickCount( tickCount ) +GraphOutputAdapter::GraphOutputAdapter( csp::Engine * engine, int32_t tickCount, TimeDelta tickHistory ) + : OutputAdapter( engine ) + , m_tickHistory( tickHistory ) + , m_tickCount( tickCount ) { } -GraphOutputAdapter::~GraphOutputAdapter() -{ -} +GraphOutputAdapter::~GraphOutputAdapter() {} -const char * GraphOutputAdapter::name() const -{ - return "GraphOutputAdapter"; -} +const char * GraphOutputAdapter::name() const { return "GraphOutputAdapter"; } void GraphOutputAdapter::start() { - //The GraphOutputAdapter is just a sync to keep the timeseries alive - //as well as set proper buffering on it. its on different language impls ( ie PyEngine ) - //to keep track of which outputs to collect when done + // The GraphOutputAdapter is just a sync to keep the timeseries alive + // as well as set proper buffering on it. its on different language impls ( ie PyEngine ) + // to keep track of which outputs to collect when done - //passive input - input() -> removeConsumer( this, InputId( 0 ) ); + // passive input + input()->removeConsumer( this, InputId( 0 ) ); bool hasTimeHistory = !m_tickHistory.isNone() && m_tickHistory > TimeDelta::ZERO(); if( m_tickCount > 0 ) - input() -> setTickCountPolicy( m_tickCount ); + input()->setTickCountPolicy( m_tickCount ); else if( m_tickCount < 0 && !hasTimeHistory ) - input() -> setTickTimeWindowPolicy( TimeDelta::MAX_VALUE() ); + input()->setTickTimeWindowPolicy( TimeDelta::MAX_VALUE() ); if( hasTimeHistory ) - input() -> setTickTimeWindowPolicy( m_tickHistory ); + input()->setTickTimeWindowPolicy( m_tickHistory ); } void GraphOutputAdapter::stop() { - //for dynamic engines the timeseries that this is referring to will most likely not exist after the GraphOutputAdapter is stopped. For this reason, - //we pre-convert the requested data up front so its cached until the end of the root engine. - if( !engine() -> isRootEngine() ) + // for dynamic engines the timeseries that this is referring to will most likely not exist after the + // GraphOutputAdapter is stopped. For this reason, we pre-convert the requested data up front so its cached until + // the end of the root engine. + if( !engine()->isRootEngine() ) processResults(); } -} +} // namespace csp diff --git a/cpp/csp/engine/InputAdapter.cpp b/cpp/csp/engine/InputAdapter.cpp index b93b379a..37ca9f0a 100644 --- a/cpp/csp/engine/InputAdapter.cpp +++ b/cpp/csp/engine/InputAdapter.cpp @@ -4,9 +4,10 @@ namespace csp { -InputAdapter::InputAdapter( Engine *engine, const CspTypePtr &type, PushMode pushMode ) : m_rootEngine( engine -> rootEngine() ), - m_pushMode( pushMode ), - m_started( false ) +InputAdapter::InputAdapter( Engine * engine, const CspTypePtr & type, PushMode pushMode ) + : m_rootEngine( engine->rootEngine() ) + , m_pushMode( pushMode ) + , m_started( false ) { if( pushMode == PushMode::BURST ) init( CspArrayType::create( type ) ); @@ -14,4 +15,4 @@ InputAdapter::InputAdapter( Engine *engine, const CspTypePtr &type, PushMode pus init( type ); } -} +} // namespace csp diff --git a/cpp/csp/engine/Node.cpp b/cpp/csp/engine/Node.cpp index 89994079..fdb79a98 100644 --- a/cpp/csp/engine/Node.cpp +++ b/cpp/csp/engine/Node.cpp @@ -6,26 +6,26 @@ namespace csp { -Node::Node( NodeDef def, - Engine * engine ) : Consumer ( engine ), - m_outputs( nullptr ), - m_def ( def ) +Node::Node( NodeDef def, Engine * engine ) + : Consumer( engine ) + , m_outputs( nullptr ) + , m_def( def ) { - m_inputs = new TimeSeriesInputBasketInfo[ m_def.numInputs ]; + m_inputs = new TimeSeriesInputBasketInfo[m_def.numInputs]; if( m_def.numOutputs > 0 ) - m_outputs = new TimeSeriesOutputBasketInfo[ m_def.numOutputs ]; + m_outputs = new TimeSeriesOutputBasketInfo[m_def.numOutputs]; } Node::~Node() { for( int i = 0; i < m_def.numInputs; ++i ) { - if( m_inputs[ i ].isSet() ) + if( m_inputs[i].isSet() ) { - //account for non-virtual dynamic basket + // account for non-virtual dynamic basket auto * basket = m_inputs[i].get(); - if( basket -> isDynamicBasket() ) + if( basket->isDynamicBasket() ) delete static_cast( basket ); else delete basket; @@ -39,9 +39,9 @@ Node::~Node() else { auto * basket = m_outputs[i].get(); - //its possible to have null outputs if we had an exception on startup before creating the output ( ie cppnode NotImplemented type ) - //check for null before checking is Basket - if( basket && basket -> isDynamicBasket() ) + // its possible to have null outputs if we had an exception on startup before creating the output ( ie + // cppnode NotImplemented type ) check for null before checking is Basket + if( basket && basket->isDynamicBasket() ) delete static_cast( basket ); else delete basket; @@ -67,15 +67,19 @@ void Node::validateOutputIndex( size_t id ) void Node::validateInputBasketSize( size_t id, size_t size ) { if( size > InputId::maxBasketElements() ) - CSP_THROW( ValueError, "Input " << id << " on node \"" << name() << "\" has basket size " << size << " which violates maximum" - << " allowable basket elements [ " << InputId::maxBasketElements() << " ]" ); + CSP_THROW( ValueError, + "Input " << id << " on node \"" << name() << "\" has basket size " << size + << " which violates maximum" + << " allowable basket elements [ " << InputId::maxBasketElements() << " ]" ); } void Node::validateOutputBasketSize( size_t id, size_t size ) { if( size > OutputId::maxBasketElements() ) - CSP_THROW( ValueError, "Output " << id << " on node \"" << name() << "\" has basket size " << size << " which violates maximum" - << " allowable basket elements [ " << OutputId::maxBasketElements() << " ]" ); + CSP_THROW( ValueError, + "Output " << id << " on node \"" << name() << "\" has basket size " << size + << " which violates maximum" + << " allowable basket elements [ " << OutputId::maxBasketElements() << " ]" ); } void Node::initInputBasket( size_t id, size_t size, bool dynamicBasket ) @@ -83,30 +87,32 @@ void Node::initInputBasket( size_t id, size_t size, bool dynamicBasket ) validateInputIndex( id ); validateInputBasketSize( id, size ); - CSP_ASSERT( !m_inputs[ id ] ); - m_inputs[ id ].set( dynamicBasket ? new DynamicInputBasketInfo( rootEngine() ) : new InputBasketInfo( rootEngine(), size ) ); + CSP_ASSERT( !m_inputs[id] ); + m_inputs[id].set( dynamicBasket ? new DynamicInputBasketInfo( rootEngine() ) + : new InputBasketInfo( rootEngine(), size ) ); } void Node::link( TimeSeriesProvider * input, InputId inputId ) { if( inputId.elemId == InputId::ELEM_ID_NONE ) { - CSP_ASSERT( !m_inputs[ inputId.id ] ); - m_inputs[ inputId.id ].set( input ); + CSP_ASSERT( !m_inputs[inputId.id] ); + m_inputs[inputId.id].set( input ); } else - inputBasket( inputId.id ) -> setElem( inputId.elemId, input ); + inputBasket( inputId.id )->setElem( inputId.elemId, input ); - input -> addConsumer( this, inputId ); + input->addConsumer( this, inputId ); } void Node::createAlarm( CspTypePtr & type, size_t id ) { - switchCspType( type, [this,&type,id]( auto tag ) + switchCspType( type, + [this, &type, id]( auto tag ) { - using T = typename decltype(tag)::type; - this -> createAlarm( type, id ); + using T = typename decltype( tag )::type; + this->createAlarm( type, id ); } ); } -} +} // namespace csp diff --git a/cpp/csp/engine/OutputAdapter.cpp b/cpp/csp/engine/OutputAdapter.cpp index c4c12f69..38bb6c8d 100644 --- a/cpp/csp/engine/OutputAdapter.cpp +++ b/cpp/csp/engine/OutputAdapter.cpp @@ -4,21 +4,20 @@ namespace csp { -OutputAdapter::OutputAdapter( csp::Engine * engine ) : Consumer( engine ), - m_input( nullptr ) +OutputAdapter::OutputAdapter( csp::Engine * engine ) + : Consumer( engine ) + , m_input( nullptr ) { } -OutputAdapter::~OutputAdapter() -{ -} +OutputAdapter::~OutputAdapter() {} void OutputAdapter::link( TimeSeriesProvider * input ) { if( m_input ) CSP_THROW( ValueError, "Attempted to link input to output adapter " << name() << " multiple times" ); m_input = input; - input -> addConsumer( this, InputId( 0 ) ); + input->addConsumer( this, InputId( 0 ) ); } -} +} // namespace csp diff --git a/cpp/csp/engine/PendingPushEvents.cpp b/cpp/csp/engine/PendingPushEvents.cpp index dd235ada..4d787a94 100644 --- a/cpp/csp/engine/PendingPushEvents.cpp +++ b/cpp/csp/engine/PendingPushEvents.cpp @@ -4,10 +4,7 @@ namespace csp { -PendingPushEvents::PendingPushEvents() -{ - -} +PendingPushEvents::PendingPushEvents() {} PendingPushEvents::~PendingPushEvents() { @@ -16,7 +13,7 @@ PendingPushEvents::~PendingPushEvents() PushEvent * e = entry.second.head; while( e ) { - PushEvent * next = e -> next; + PushEvent * next = e->next; delete e; e = next; } @@ -27,7 +24,7 @@ PendingPushEvents::~PendingPushEvents() PushEvent * e = entry.second.head; while( e ) { - PushEvent * next = e -> next; + PushEvent * next = e->next; delete e; e = next; } @@ -36,43 +33,43 @@ PendingPushEvents::~PendingPushEvents() void PendingPushEvents::addPendingEvent( PushEvent * event ) { - PushInputAdapter * adapter = event -> adapter(); - event -> next = nullptr; + PushInputAdapter * adapter = event->adapter(); + event->next = nullptr; - if( adapter -> group() ) + if( adapter->group() ) { - auto rv = m_groupedEvents.emplace( adapter -> group(), EventList{ event, event } ); + auto rv = m_groupedEvents.emplace( adapter->group(), EventList{ event, event } ); if( !rv.second ) { - rv.first -> second.tail -> next = event; - rv.first -> second.tail = event; + rv.first->second.tail->next = event; + rv.first->second.tail = event; } } else { - assert( adapter -> pushMode() == PushMode::NON_COLLAPSING ); + assert( adapter->pushMode() == PushMode::NON_COLLAPSING ); auto rv = m_ungroupedEvents.emplace( adapter, EventList{ event, event } ); if( !rv.second ) { - rv.first -> second.tail -> next = event; - rv.first -> second.tail = event; + rv.first->second.tail->next = event; + rv.first->second.tail = event; } } } -void PendingPushEvents::processPendingEvents( std::vector & dirtyGroups ) +void PendingPushEvents::processPendingEvents( std::vector & dirtyGroups ) { for( auto it = m_ungroupedEvents.begin(); it != m_ungroupedEvents.end(); ) { - //for ungrouped events we'll never process more than one at a time - auto * event = it -> second.head; - it -> second.head = event -> next; - bool rv = it -> first -> consumeEvent( event, dirtyGroups ); + // for ungrouped events we'll never process more than one at a time + auto * event = it->second.head; + it->second.head = event->next; + bool rv = it->first->consumeEvent( event, dirtyGroups ); (void)rv; assert( rv == true ); - if( it -> second.head == nullptr ) + if( it->second.head == nullptr ) it = m_ungroupedEvents.erase( it ); else ++it; @@ -80,29 +77,29 @@ void PendingPushEvents::processPendingEvents( std::vector & dirtyGro for( auto it = m_groupedEvents.begin(); it != m_groupedEvents.end(); ) { - auto * group = it -> first; - assert( group -> state == PushGroup::NONE ); + auto * group = it->first; + assert( group->state == PushGroup::NONE ); - auto * event = it -> second.head; + auto * event = it->second.head; PushEvent * deferred_head = nullptr; PushEvent * deferred_tail = nullptr; - while( event && group -> state != PushGroup::LOCKED ) + while( event && group->state != PushGroup::LOCKED ) { - PushEvent * next = event -> next; + PushEvent * next = event->next; - bool consumed = event -> adapter() -> consumeEvent( event, dirtyGroups ); + bool consumed = event->adapter()->consumeEvent( event, dirtyGroups ); if( !consumed ) { - if( !deferred_head ) + if( !deferred_head ) { deferred_head = event; deferred_tail = event; } else { - deferred_tail -> next = event; - deferred_tail = event; + deferred_tail->next = event; + deferred_tail = event; } } @@ -111,19 +108,19 @@ void PendingPushEvents::processPendingEvents( std::vector & dirtyGro if( unlikely( deferred_head != nullptr ) ) { - deferred_tail -> flagGroupEnd(); //ensure we flag group end on the last deferred events in this batch - deferred_tail -> next = event; //ensure we link to the next batch - event = deferred_head; //ensure deferred list gets executes next cycle + deferred_tail->flagGroupEnd(); // ensure we flag group end on the last deferred events in this batch + deferred_tail->next = event; // ensure we link to the next batch + event = deferred_head; // ensure deferred list gets executes next cycle } if( event == nullptr ) it = m_groupedEvents.erase( it ); else { - it -> second.head = event; + it->second.head = event; ++it; } } } -} +} // namespace csp diff --git a/cpp/csp/engine/PushPullInputAdapter.cpp b/cpp/csp/engine/PushPullInputAdapter.cpp index 051b0a20..4c05796c 100644 --- a/cpp/csp/engine/PushPullInputAdapter.cpp +++ b/cpp/csp/engine/PushPullInputAdapter.cpp @@ -2,14 +2,14 @@ namespace csp { -PushPullInputAdapter::PushPullInputAdapter( Engine *engine, CspTypePtr &type, PushMode pushMode, - PushGroup *group, bool adjustOutOfOrderTime ) - : PushInputAdapter(engine, type, pushMode, group), - m_nextPullEvent(nullptr), - m_notifiedEndOfPull(false), - m_adjustOutOfOrderTime(adjustOutOfOrderTime) +PushPullInputAdapter::PushPullInputAdapter( Engine * engine, CspTypePtr & type, PushMode pushMode, PushGroup * group, + bool adjustOutOfOrderTime ) + : PushInputAdapter( engine, type, pushMode, group ) + , m_nextPullEvent( nullptr ) + , m_notifiedEndOfPull( false ) + , m_adjustOutOfOrderTime( adjustOutOfOrderTime ) { - //free up any unused events + // free up any unused events while( m_nextPullEvent ) { delete m_nextPullEvent; @@ -22,48 +22,48 @@ void PushPullInputAdapter::start( DateTime start, DateTime end ) m_nextPullEvent = nextPullEvent(); if( m_nextPullEvent ) { - m_timerHandle = rootEngine() -> scheduleCallback( m_nextPullEvent -> time, - [this]() { return processNextPullEvent() ? nullptr : this; } ); + m_timerHandle = rootEngine()->scheduleCallback( m_nextPullEvent->time, + [this]() { return processNextPullEvent() ? nullptr : this; } ); } } void PushPullInputAdapter::stop() { - rootEngine() -> cancelCallback( m_timerHandle ); - //shouldnt need to lock at this point + rootEngine()->cancelCallback( m_timerHandle ); + // shouldnt need to lock at this point m_threadQueue.emplace( nullptr ); } bool PushPullInputAdapter::processNextPullEvent() { - bool consumed = switchCspType( dataType(), - [ this ]( auto tag ) - { - using T = typename decltype(tag)::type; - TypedPullDataEvent *tevent = static_cast *>( m_nextPullEvent ); + bool consumed = switchCspType( + dataType(), + [this]( auto tag ) + { + using T = typename decltype( tag )::type; + TypedPullDataEvent * tevent = static_cast *>( m_nextPullEvent ); - bool consumed = consumeTick( tevent -> data ); - assert( consumed ); + bool consumed = consumeTick( tevent->data ); + assert( consumed ); - delete tevent; + delete tevent; - while( ( m_nextPullEvent = nextPullEvent() ) && - m_nextPullEvent -> time == rootEngine() -> now() ) - { - tevent = static_cast *>( m_nextPullEvent ); - consumed = consumeTick( tevent -> data ); - if( !consumed ) - return false; - delete tevent; - } + while( ( m_nextPullEvent = nextPullEvent() ) && m_nextPullEvent->time == rootEngine()->now() ) + { + tevent = static_cast *>( m_nextPullEvent ); + consumed = consumeTick( tevent->data ); + if( !consumed ) + return false; + delete tevent; + } - return true; - } ); + return true; + } ); if( consumed && m_nextPullEvent ) { - m_timerHandle = rootEngine() -> scheduleCallback( m_nextPullEvent->time, - [this]() { return processNextPullEvent() ? nullptr : this; } ); + m_timerHandle = rootEngine()->scheduleCallback( m_nextPullEvent->time, + [this]() { return processNextPullEvent() ? nullptr : this; } ); } return consumed; @@ -71,7 +71,7 @@ bool PushPullInputAdapter::processNextPullEvent() PushPullInputAdapter::PullDataEvent * PushPullInputAdapter::nextPullEvent() { - //spin while we wait for data + // spin while we wait for data while( m_poppedPullEvents.empty() ) { std::lock_guard g( m_queueMutex ); @@ -82,9 +82,9 @@ PushPullInputAdapter::PullDataEvent * PushPullInputAdapter::nextPullEvent() m_poppedPullEvents.pop(); if( m_adjustOutOfOrderTime && event ) - event -> time = std::max( event -> time, rootEngine() -> now() ); + event->time = std::max( event->time, rootEngine()->now() ); - return event; + return event; } -} +} // namespace csp diff --git a/cpp/csp/engine/RootEngine.cpp b/cpp/csp/engine/RootEngine.cpp index cc0beca9..5893941c 100644 --- a/cpp/csp/engine/RootEngine.cpp +++ b/cpp/csp/engine/RootEngine.cpp @@ -1,17 +1,17 @@ -#include #include -#include +#include #include #include #include #include -#include +#include #include +#include namespace csp { -static volatile int g_SIGNAL_COUNT = 0; +static volatile int g_SIGNAL_COUNT = 0; /* The signal count variable is maintained to ensure that multiple engine threads shutdown properly. @@ -24,14 +24,14 @@ Future runs after the interrupt remain unaffected since they are initialized wit only consider themselves "interupted" if another signal is received during their execution. */ -static void (*g_prevSIGTERMhandler)(int) = nullptr; +static void ( *g_prevSIGTERMhandler )( int ) = nullptr; static void handle_SIGTERM( int signum ) { g_SIGNAL_COUNT++; - if(g_prevSIGTERMhandler) - (*g_prevSIGTERMhandler)( signum ); + if( g_prevSIGTERMhandler ) + ( *g_prevSIGTERMhandler )( signum ); } static bool install_signal_handlers() @@ -47,13 +47,13 @@ static bool install_signal_handlers() struct sigaction newhandler, prev_handler; sigemptyset( &newhandler.sa_mask ); newhandler.sa_handler = handle_SIGTERM; - newhandler.sa_flags = 0; + newhandler.sa_flags = 0; - if( sigaction(SIGINT,&newhandler, &prev_handler) != 0 ) + if( sigaction( SIGINT, &newhandler, &prev_handler ) != 0 ) printf( "Failed to set SIGTERM handler: %s", strerror( errno ) ); g_prevSIGTERMhandler = prev_handler.sa_handler; #else - g_prevSIGTERMhandler = signal(SIGINT, handle_SIGTERM); + g_prevSIGTERMhandler = signal( SIGINT, handle_SIGTERM ); #endif s_installed = true; } @@ -66,51 +66,46 @@ static bool s_install_handlers = install_signal_handlers(); RootEngine::Settings::Settings( const Dictionary & settings ) { queueWaitTime = settings.get( "queue_wait_time", TimeDelta::fromMilliseconds( 100 ) ); - realtime = settings.get( "realtime", false ); + realtime = settings.get( "realtime", false ); } -RootEngine::RootEngine( const Dictionary & settings ) : Engine( m_cycleStepTable ), - m_now( DateTime::NONE() ), - m_state( State::NONE ), - m_cycleCount( 0 ), - m_settings( settings ), - m_inRealtime( false ), - m_initSignalCount( g_SIGNAL_COUNT ), - m_pushEventQueue( m_settings.queueWaitTime > TimeDelta::ZERO() ) +RootEngine::RootEngine( const Dictionary & settings ) + : Engine( m_cycleStepTable ) + , m_now( DateTime::NONE() ) + , m_state( State::NONE ) + , m_cycleCount( 0 ) + , m_settings( settings ) + , m_inRealtime( false ) + , m_initSignalCount( g_SIGNAL_COUNT ) + , m_pushEventQueue( m_settings.queueWaitTime > TimeDelta::ZERO() ) { - if( settings.get( "profile", false ) ) + if( settings.get( "profile", false ) ) { - m_profiler = std::make_unique(); + m_profiler = std::make_unique(); std::string cycle_fname = settings.get( "cycle_profile_file", "" ); - std::string node_fname = settings.get( "node_profile_file", "" ); + std::string node_fname = settings.get( "node_profile_file", "" ); if( !cycle_fname.empty() ) - m_profiler.get() -> use_prof_file( cycle_fname, false ); + m_profiler.get()->use_prof_file( cycle_fname, false ); if( !node_fname.empty() ) - m_profiler.get() -> use_prof_file( node_fname, true ); + m_profiler.get()->use_prof_file( node_fname, true ); } } -RootEngine::~RootEngine() -{ -} +RootEngine::~RootEngine() {} -bool RootEngine::interrupted() const -{ - return g_SIGNAL_COUNT != m_initSignalCount; -} +bool RootEngine::interrupted() const { return g_SIGNAL_COUNT != m_initSignalCount; } void RootEngine::preRun( DateTime start, DateTime end ) { m_state = State::STARTING; - m_now = start; + m_now = start; m_startTime = start; - m_endTime = end; + m_endTime = end; Engine::start(); } - void RootEngine::postRun() { m_state = State::SHUTDOWN; @@ -126,9 +121,9 @@ void RootEngine::processPushEventQueue( PushEvent * event, std::vector next; + PushEvent * next = event->next; - if( !event -> adapter() -> consumeEvent( event, dirtyGroups ) ) + if( !event->adapter()->consumeEvent( event, dirtyGroups ) ) m_pendingPushEvents.addPendingEvent( event ); event = next; @@ -139,8 +134,8 @@ void RootEngine::processEndCycle() { for( auto * listener : m_endCycleListeners ) { - listener -> onEndCycle(); - listener -> clearDirtyFlag(); + listener->onEndCycle(); + listener->clearDirtyFlag(); } m_endCycleListeners.clear(); @@ -170,14 +165,18 @@ void RootEngine::runRealtime( DateTime end ) { struct DialectReleaseGIL { - DialectReleaseGIL( RootEngine * e ) : engine( e ) { engine -> dialectUnlockGIL(); } - ~DialectReleaseGIL() { engine -> dialectLockGIL(); } + DialectReleaseGIL( RootEngine * e ) + : engine( e ) + { + engine->dialectUnlockGIL(); + } + ~DialectReleaseGIL() { engine->dialectLockGIL(); } RootEngine * engine; }; std::vector dirtyGroups; - m_inRealtime = true; + m_inRealtime = true; bool haveEvents = false; while( m_state == State::RUNNING && !interrupted() ) { @@ -185,29 +184,29 @@ void RootEngine::runRealtime( DateTime end ) if( !m_pendingPushEvents.hasEvents() ) { DateTime now = DateTime::now(); - waitTime = std::min( m_endTime - now, m_settings.queueWaitTime ); + waitTime = std::min( m_endTime - now, m_settings.queueWaitTime ); if( m_scheduler.hasEvents() ) waitTime = std::min( m_scheduler.nextTime() - DateTime::now(), waitTime ); } if( !haveEvents ) { - //We keep the haveEvents flag in case there were events, but we only decided to execute - //timers in the previous cycle, then we shouldnt wait again ( which can actually lead to cases - //where we miss triggers ) + // We keep the haveEvents flag in case there were events, but we only decided to execute + // timers in the previous cycle, then we shouldnt wait again ( which can actually lead to cases + // where we miss triggers ) DialectReleaseGIL release( this ); haveEvents = m_pushEventQueue.wait( waitTime ); } - //grab time after waitForEvents so that we dont grab events with time > now + // grab time after waitForEvents so that we dont grab events with time > now m_now = DateTime::now(); if( m_now > end ) break; ++m_cycleCount; - //We made a conscious decision to execute timers exactly on their requested time in realtime mode - //therefore timers that are ready are executed on their own time cycle before realtime events all processed + // We made a conscious decision to execute timers exactly on their requested time in realtime mode + // therefore timers that are ready are executed on their own time cycle before realtime events all processed if( m_scheduler.hasEvents() && m_scheduler.nextTime() < m_now ) { m_now = m_scheduler.nextTime(); @@ -221,8 +220,8 @@ void RootEngine::runRealtime( DateTime end ) processPushEventQueue( events, dirtyGroups ); for( auto * group : dirtyGroups ) - group -> state = PushGroup::NONE; - + group->state = PushGroup::NONE; + dirtyGroups.clear(); haveEvents = false; } @@ -276,10 +275,7 @@ void RootEngine::run( DateTime start, DateTime end ) std::rethrow_exception( m_exception_ptr ); } -void RootEngine::shutdown() -{ - m_state = State::SHUTDOWN; -} +void RootEngine::shutdown() { m_state = State::SHUTDOWN; } void RootEngine::shutdown( std::exception_ptr except_ptr ) { @@ -294,8 +290,7 @@ DictionaryPtr RootEngine::engine_stats() const if( !m_profiler ) CSP_THROW( ValueError, "Cannot profile a graph unless the graph is run in a profiler context." ); - return m_profiler -> getAllStats( m_nodes.size() ); + return m_profiler->getAllStats( m_nodes.size() ); } - -} +} // namespace csp diff --git a/cpp/csp/engine/Scheduler.cpp b/cpp/csp/engine/Scheduler.cpp index 368ec0a8..a5d893a9 100644 --- a/cpp/csp/engine/Scheduler.cpp +++ b/cpp/csp/engine/Scheduler.cpp @@ -9,9 +9,11 @@ namespace csp #define debug_printf( ... ) #endif -Scheduler::Scheduler() : m_pendingEvents( *this ), m_unique( 0 ) +Scheduler::Scheduler() + : m_pendingEvents( *this ) + , m_unique( 0 ) { - //should make configurable at some point + // should make configurable at some point m_eventAllocator.init( 32768, false, true ); } @@ -19,11 +21,11 @@ Scheduler::~Scheduler() { for( auto it = m_map.begin(); it != m_map.end(); ++it ) { - Event * event = it -> second.head; + Event * event = it->second.head; while( event ) { - Event * nextEvent = event -> next; - event -> ~Event(); + Event * nextEvent = event->next; + event->~Event(); event = nextEvent; } } @@ -31,26 +33,25 @@ Scheduler::~Scheduler() m_pendingEvents.clear(); } -//PendingEvents processing -Scheduler::PendingEvents::PendingEvents( Scheduler & scheduler ) : m_scheduler( scheduler ) +// PendingEvents processing +Scheduler::PendingEvents::PendingEvents( Scheduler & scheduler ) + : m_scheduler( scheduler ) { } -Scheduler::PendingEvents::~PendingEvents() -{ -} +Scheduler::PendingEvents::~PendingEvents() {} void Scheduler::PendingEvents::clear() { for( auto it = m_pendingEvents.begin(); it != m_pendingEvents.end(); it++ ) { // skip past the sentinel, which is on the stack - Event * event = it -> head.next; + Event * event = it->head.next; - while( event != &it -> tail ) + while( event != &it->tail ) { - Event * nextEvent = event -> next; - event -> ~Event(); + Event * nextEvent = event->next; + event->~Event(); event = nextEvent; } } @@ -58,10 +59,10 @@ void Scheduler::PendingEvents::clear() void Scheduler::PendingEvents::addPendingEvent( const InputAdapter * adapter, Event * event, DateTime now ) { - //reset mapIt so event knows its no longer in main scheduler list ( needed for cancel handling ) - event -> mapIt = m_scheduler.m_map.end(); + // reset mapIt so event knows its no longer in main scheduler list ( needed for cancel handling ) + event->mapIt = m_scheduler.m_map.end(); - //unconditionally overwrite, no need for conditional check here + // unconditionally overwrite, no need for conditional check here assert( m_pendingAdapters.empty() || m_time == now ); m_time = now; @@ -70,85 +71,84 @@ void Scheduler::PendingEvents::addPendingEvent( const InputAdapter * adapter, Ev auto rv = m_pendingAdapters.emplace( adapter, std::list::iterator() ); if( rv.second ) { - auto peIt = m_pendingEvents.emplace( m_pendingEvents.end() ); - rv.first -> second = peIt; + auto peIt = m_pendingEvents.emplace( m_pendingEvents.end() ); + rv.first->second = peIt; PendingEventList & el = *peIt; - el.adapter = adapter; - el.head.next = &el.tail; - el.tail.prev = &el.head; + el.adapter = adapter; + el.head.next = &el.tail; + el.tail.prev = &el.head; debug_printf( "new entry for adapter %p\n", adapter ); } - auto & el = *rv.first -> second; + auto & el = *rv.first->second; - el.tail.prev -> next = event; - event -> prev = el.tail.prev; + el.tail.prev->next = event; + event->prev = el.tail.prev; el.tail.prev = event; - event -> next = &el.tail; + event->next = &el.tail; } void Scheduler::PendingEvents::cancelEvent( Event * event ) { - //we maintain sentinel head/tail values so that we can just erase in place without much thought - event -> prev -> next = event -> next; - event -> next -> prev = event -> prev; + // we maintain sentinel head/tail values so that we can just erase in place without much thought + event->prev->next = event->next; + event->next->prev = event->prev; } void Scheduler::PendingEvents::executeCycle() { debug_printf( "Scheduler::PendingEvents::executeCycle\n" ); - //execute one event per pending adapter + // execute one event per pending adapter for( auto elt = m_pendingEvents.begin(); elt != m_pendingEvents.end(); ) { - //It could be that we have an empty list due to a cancel of the last event - Event * event = elt -> head.next; + // It could be that we have an empty list due to a cancel of the last event + Event * event = elt->head.next; - debug_printf( "Processing pending event %p on adapter %p\n", event, elt -> adapter ); - if( unlikely( event == &elt -> tail ) ) + debug_printf( "Processing pending event %p on adapter %p\n", event, elt->adapter ); + if( unlikely( event == &elt->tail ) ) { elt = m_pendingEvents.erase( elt ); continue; } - auto eventId = event -> id; - event -> id = -1; + auto eventId = event->id; + event->id = -1; - if( unlikely( event -> func() != nullptr ) ) + if( unlikely( event->func() != nullptr ) ) { - //Note its possible this can return false, even though we do one at a time - //This is how PullInputAdapter is already implemented, it keeps consuming if next returns the same timestamp ( needed for bursts ) - //If that is ever reworked, we can add an exception for returning false here - //CSP_THROW( RuntimeException, "Logical error process pending schedule events" ); - event -> id = eventId; + // Note its possible this can return false, even though we do one at a time + // This is how PullInputAdapter is already implemented, it keeps consuming if next returns the same + // timestamp ( needed for bursts ) If that is ever reworked, we can add an exception for returning false + // here CSP_THROW( RuntimeException, "Logical error process pending schedule events" ); + event->id = eventId; ++elt; continue; } - //list is done - if( event -> next == &elt -> tail ) + // list is done + if( event->next == &elt->tail ) { - debug_printf( "end of list, removing adapter %p\n", elt -> adapter ); - m_pendingAdapters.erase( elt -> adapter ); + debug_printf( "end of list, removing adapter %p\n", elt->adapter ); + m_pendingAdapters.erase( elt->adapter ); elt = m_pendingEvents.erase( elt ); } else { - debug_printf( "events remaining on adapter %p, setting head -> next to %p\n", elt -> adapter, event -> next ); - elt -> head.next = event -> next; - event -> next -> prev = &elt -> head; + debug_printf( "events remaining on adapter %p, setting head -> next to %p\n", elt->adapter, event->next ); + elt->head.next = event->next; + event->next->prev = &elt->head; ++elt; } - event -> ~Event(); + event->~Event(); m_scheduler.m_eventAllocator.free( event ); } } - -//Regular non-pending handling +// Regular non-pending handling bool Scheduler::cancelCallback( Handle handle ) { if( handle.expired() ) @@ -156,37 +156,37 @@ bool Scheduler::cancelCallback( Handle handle ) Event * event = handle.event; - //in pending events lists, no longer under m_maps domain - if( event -> mapIt == m_map.end() ) + // in pending events lists, no longer under m_maps domain + if( event->mapIt == m_map.end() ) { m_pendingEvents.cancelEvent( event ); } - //now back to your regularly scheduled program + // now back to your regularly scheduled program else { - //only event at this time slice - if( event -> prev == nullptr && event -> next == nullptr ) + // only event at this time slice + if( event->prev == nullptr && event->next == nullptr ) { - m_map.erase( event -> mapIt ); + m_map.erase( event->mapIt ); } else { - EventList * list = &event -> mapIt -> second; - if( event -> next ) - event -> next -> prev = event -> prev; - else //else we are tail - list -> tail = event -> prev; - - if( event -> prev ) - event -> prev -> next = event -> next; - else //else we are head - list -> head = event -> next; + EventList * list = &event->mapIt->second; + if( event->next ) + event->next->prev = event->prev; + else // else we are tail + list->tail = event->prev; + + if( event->prev ) + event->prev->next = event->next; + else // else we are head + list->head = event->next; } } - event -> ~Event(); + event->~Event(); m_eventAllocator.free( event ); - event -> id = -1; + event->id = -1; return true; } @@ -196,13 +196,13 @@ void Scheduler::executeNextEvents( DateTime now, Event * start ) if( unlikely( m_pendingEvents.hasEvents() ) ) m_pendingEvents.executeCycle(); - if( m_map.empty() || m_map.begin() -> first > now ) + if( m_map.empty() || m_map.begin()->first > now ) return; - EventList & elist = m_map.begin() -> second; + EventList & elist = m_map.begin()->second; // startPrev kept locally here since start wil be freed below - Event * startPrev = unlikely( start != nullptr ) ? start -> prev : nullptr; + Event * startPrev = unlikely( start != nullptr ) ? start->prev : nullptr; // Note event is kept as a reference so that it gets updated throughout the loop Event *& event = unlikely( start != nullptr ) ? start : elist.head; // mark end before iterating in case timers with same time are added during loop @@ -210,24 +210,24 @@ void Scheduler::executeNextEvents( DateTime now, Event * start ) while( true ) { - //invalidate event / handle before calling in case cb decides to reschedule itself - auto eventId = event -> id; - event -> id = -1; - Event * cur = event; + // invalidate event / handle before calling in case cb decides to reschedule itself + auto eventId = event->id; + event->id = -1; + Event * cur = event; debug_printf( "Main loop processing event %p with id %lu\n", cur, eventId ); - auto * deferredAdapter = cur -> func(); - event = cur -> next; + auto * deferredAdapter = cur->func(); + event = cur->next; if( unlikely( deferredAdapter != nullptr ) ) { - //set id back to what it was, alarm is still cancellable - cur -> id = eventId; + // set id back to what it was, alarm is still cancellable + cur->id = eventId; m_pendingEvents.addPendingEvent( deferredAdapter, cur, now ); } else { - cur -> ~Event(); + cur->~Event(); m_eventAllocator.free( cur ); } @@ -235,36 +235,36 @@ void Scheduler::executeNextEvents( DateTime now, Event * start ) break; } - //got events for same time during loop + // got events for same time during loop if( event ) { - //if we started from the middle of the list, link from where we started, otherwise we would miss these events - //Note that start is only set when starting dynamic engines + // if we started from the middle of the list, link from where we started, otherwise we would miss these events + // Note that start is only set when starting dynamic engines if( startPrev ) - event -> prev = startPrev; + event->prev = startPrev; } else if( startPrev ) { - //if we have startPrev that means this is a dynamic engine start flush of events scheduled for "now" in engine - //construction. in this case we want to ensure the dynamic events are no longer in the queue - startPrev -> next = nullptr; - elist.tail = startPrev; + // if we have startPrev that means this is a dynamic engine start flush of events scheduled for "now" in engine + // construction. in this case we want to ensure the dynamic events are no longer in the queue + startPrev->next = nullptr; + elist.tail = startPrev; } else m_map.erase( m_map.begin() ); } -Scheduler::DynamicEngineStartMonitor::DynamicEngineStartMonitor( Scheduler & s, - DateTime now ) : m_scheduler( s ), - m_now( now ) +Scheduler::DynamicEngineStartMonitor::DynamicEngineStartMonitor( Scheduler & s, DateTime now ) + : m_scheduler( s ) + , m_now( now ) { - //note we only keep track of main map time, not pending events, since all initial dynamic events - //will schedule into main - DateTime nextMapTime = m_scheduler.m_map.begin() -> first; + // note we only keep track of main map time, not pending events, since all initial dynamic events + // will schedule into main + DateTime nextMapTime = m_scheduler.m_map.begin()->first; if( m_scheduler.hasEvents() && nextMapTime == now ) { - m_lastEvent = m_scheduler.m_map.begin() -> second.tail; - assert( m_lastEvent -> next == nullptr ); + m_lastEvent = m_scheduler.m_map.begin()->second.tail; + assert( m_lastEvent->next == nullptr ); } else m_lastEvent = nullptr; @@ -273,20 +273,19 @@ Scheduler::DynamicEngineStartMonitor::DynamicEngineStartMonitor( Scheduler & s, Scheduler::DynamicEngineStartMonitor::~DynamicEngineStartMonitor() { - DateTime nextMapTime = m_scheduler.m_map.begin() -> first; - if( m_scheduler.hasEvents() && nextMapTime == m_now && - ( !m_lastEvent || //we had no events at "now" previously - m_lastEvent -> next != nullptr ) // or there were events and we added new ones - ) + DateTime nextMapTime = m_scheduler.m_map.begin()->first; + if( m_scheduler.hasEvents() && nextMapTime == m_now + && ( !m_lastEvent || // we had no events at "now" previously + m_lastEvent->next != nullptr ) // or there were events and we added new ones + ) { - Event * startEvent = m_lastEvent ? m_lastEvent -> next : nullptr; + Event * startEvent = m_lastEvent ? m_lastEvent->next : nullptr; debug_printf( "DynamicEngineStartMonitor::~DynamicEngineStartMonitor() startEvent %p\n", startEvent ); m_scheduler.executeNextEvents( m_now, startEvent ); } else debug_printf( "DynamicEngineStartMonitor::~DynamicEngineStartMonitor() no events\n" ); - } -} +} // namespace csp diff --git a/cpp/csp/engine/StatusAdapter.cpp b/cpp/csp/engine/StatusAdapter.cpp index 68cc486e..3beb644c 100644 --- a/cpp/csp/engine/StatusAdapter.cpp +++ b/cpp/csp/engine/StatusAdapter.cpp @@ -3,12 +3,6 @@ namespace csp { -INIT_CSP_ENUM( csp::StatusLevel, - "DEBUG", - "INFO", - "WARNING", - "ERROR", - "CRITICAL" -); +INIT_CSP_ENUM( csp::StatusLevel, "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL" ); } \ No newline at end of file diff --git a/cpp/csp/engine/Struct.cpp b/cpp/csp/engine/Struct.cpp index 837d519b..490ebb48 100644 --- a/cpp/csp/engine/Struct.cpp +++ b/cpp/csp/engine/Struct.cpp @@ -1,24 +1,23 @@ +#include #include #include -#include namespace csp { -StructField::StructField( CspTypePtr type, const std::string & fieldname, - size_t size, size_t alignment ) : - m_fieldname( fieldname ), - m_offset( 0 ), - m_size( size ), - m_alignment( alignment ), - m_maskOffset( 0 ), - m_maskBit( 0 ), - m_maskBitMask( 0 ), - m_type( type ) +StructField::StructField( CspTypePtr type, const std::string & fieldname, size_t size, size_t alignment ) + : m_fieldname( fieldname ) + , m_offset( 0 ) + , m_size( size ) + , m_alignment( alignment ) + , m_maskOffset( 0 ) + , m_maskBit( 0 ) + , m_maskBitMask( 0 ) + , m_type( type ) { } -/* StructMeta +/* StructMeta A note on member layout. Meta will order objects in the following order: - non-native fields ( ie PyObjectPtr for the python dialect ) @@ -26,85 +25,92 @@ A note on member layout. Meta will order objects in the following order: - set/unset bitmask bytes ( 1 byte per 8 fields ) Derived structs will simply append to the layout of the base struct, properly padding between classes to align -its fields properly. -This layout is imposed on Struct instances. Since Struct needs refcount and meta * fields, for convenience they are stored -*before* a Struct's "this" pointer as hidden data. This way struct ptrs can be passed into StructMeta without +its fields properly. +This layout is imposed on Struct instances. Since Struct needs refcount and meta * fields, for convenience they are +stored *before* a Struct's "this" pointer as hidden data. This way struct ptrs can be passed into StructMeta without and adjustments required for the hidden fields */ -StructMeta::StructMeta( const std::string & name, const Fields & fields, - std::shared_ptr base ) : m_name( name ), m_base( base ), m_fields( fields ), - m_size( 0 ), m_partialSize( 0 ), m_partialStart( 0 ), m_nativeStart( 0 ), m_basePadding( 0 ), - m_maskLoc( 0 ), m_maskSize( 0 ), m_firstPartialField( 0 ), m_firstNativePartialField( 0 ), - m_isPartialNative( true ), m_isFullyNative( true ) +StructMeta::StructMeta( const std::string & name, const Fields & fields, std::shared_ptr base ) + : m_name( name ) + , m_base( base ) + , m_fields( fields ) + , m_size( 0 ) + , m_partialSize( 0 ) + , m_partialStart( 0 ) + , m_nativeStart( 0 ) + , m_basePadding( 0 ) + , m_maskLoc( 0 ) + , m_maskSize( 0 ) + , m_firstPartialField( 0 ) + , m_firstNativePartialField( 0 ) + , m_isPartialNative( true ) + , m_isFullyNative( true ) { - if( m_fields.empty() && !m_base) + if( m_fields.empty() && !m_base ) CSP_THROW( TypeError, "Struct types must define at least 1 field" ); - //sort by sizes, biggest first, to get proper alignment - //group generic objects separately at the start so that we can safely memcpy native types - //decided to place them at the start cause they are most likely size of ptr or greater + // sort by sizes, biggest first, to get proper alignment + // group generic objects separately at the start so that we can safely memcpy native types + // decided to place them at the start cause they are most likely size of ptr or greater m_fieldnames.reserve( m_fields.size() ); - for( size_t i = 0; i < m_fields.size(); i++ ) - m_fieldnames.emplace_back( m_fields[i] -> fieldname() ); + for( size_t i = 0; i < m_fields.size(); i++ ) + m_fieldnames.emplace_back( m_fields[i]->fieldname() ); - std::sort( m_fields.begin(), m_fields.end(), []( auto && a, auto && b ) - { - return a -> isNative() < b -> isNative() || - a -> size() > b -> size(); - } ); + std::sort( m_fields.begin(), m_fields.end(), + []( auto && a, auto && b ) { return a->isNative() < b->isNative() || a->size() > b->size(); } ); - size_t baseSize = m_base ? m_base -> size() : 0; - size_t offset = baseSize; - m_basePadding = 0; + size_t baseSize = m_base ? m_base->size() : 0; + size_t offset = baseSize; + m_basePadding = 0; - //align to first field's alignment - if( m_fields.size() && ( offset % m_fields[0] -> alignment() != 0 ) ) - m_basePadding = m_fields[0] -> alignment() - offset % m_fields[0] -> alignment(); + // align to first field's alignment + if( m_fields.size() && ( offset % m_fields[0]->alignment() != 0 ) ) + m_basePadding = m_fields[0]->alignment() - offset % m_fields[0]->alignment(); offset += m_basePadding; if( !m_fields.empty() ) - CSP_ASSERT( ( offset % m_fields[0] -> alignment() ) == 0 ); + CSP_ASSERT( ( offset % m_fields[0]->alignment() ) == 0 ); m_partialStart = offset; m_nativeStart = m_partialStart; for( size_t idx = 0; idx < m_fields.size(); ++idx ) { - auto & f = m_fields[ idx ]; - if( offset % f -> alignment() != 0 ) - offset += f -> alignment() - offset % f -> alignment(); + auto & f = m_fields[idx]; + if( offset % f->alignment() != 0 ) + offset += f->alignment() - offset % f->alignment(); - f -> setOffset( offset ); + f->setOffset( offset ); - offset += f -> size(); + offset += f->size(); - m_isPartialNative &= f -> isNative(); + m_isPartialNative &= f->isNative(); - if( !f -> isNative() ) + if( !f->isNative() ) { - m_nativeStart = offset; + m_nativeStart = offset; m_firstNativePartialField = idx + 1; } } - m_isFullyNative = m_isPartialNative && ( m_base ? m_base -> isNative() : true ); + m_isFullyNative = m_isPartialNative && ( m_base ? m_base->isNative() : true ); - //Setup masking bits for our fields - //NOTE we can be more efficient by sticking masks into any potential alignment gaps, dont want to spend time on it - //at this point - m_maskSize = !m_fields.empty() ? 1 + ( ( m_fields.size() - 1 ) / 8 ) : 0; - m_size = offset + m_maskSize; - m_partialSize = m_size - baseSize; - m_maskLoc = m_size - m_maskSize; + // Setup masking bits for our fields + // NOTE we can be more efficient by sticking masks into any potential alignment gaps, dont want to spend time on it + // at this point + m_maskSize = !m_fields.empty() ? 1 + ( ( m_fields.size() - 1 ) / 8 ) : 0; + m_size = offset + m_maskSize; + m_partialSize = m_size - baseSize; + m_maskLoc = m_size - m_maskSize; - size_t maskLoc = m_maskLoc; + size_t maskLoc = m_maskLoc; uint8_t maskBit = 0; for( auto & f : m_fields ) { - f -> setMaskOffset( maskLoc, maskBit ); + f->setMaskOffset( maskLoc, maskBit ); if( ++maskBit == 8 ) { maskBit = 0; @@ -114,38 +120,36 @@ StructMeta::StructMeta( const std::string & name, const Fields & fields, if( m_base ) { - m_fields.insert( m_fields.begin(), m_base -> m_fields.begin(), m_base -> m_fields.end() ); - m_fieldnames.insert( m_fieldnames.begin(), m_base -> m_fieldnames.begin(), m_base -> m_fieldnames.end() ); - - m_firstPartialField = m_base -> m_fields.size(); - m_firstNativePartialField += m_base -> m_fields.size(); - m_fieldMap = m_base -> m_fieldMap; + m_fields.insert( m_fields.begin(), m_base->m_fields.begin(), m_base->m_fields.end() ); + m_fieldnames.insert( m_fieldnames.begin(), m_base->m_fieldnames.begin(), m_base->m_fieldnames.end() ); + + m_firstPartialField = m_base->m_fields.size(); + m_firstNativePartialField += m_base->m_fields.size(); + m_fieldMap = m_base->m_fieldMap; } for( size_t idx = m_firstPartialField; idx < m_fields.size(); ++idx ) { - auto rv = m_fieldMap.emplace( m_fields[ idx ] -> fieldname().c_str(), m_fields[ idx ] ); + auto rv = m_fieldMap.emplace( m_fields[idx]->fieldname().c_str(), m_fields[idx] ); if( !rv.second ) - CSP_THROW( ValueError, "csp Struct " << name << " attempted to add existing field " << m_fields[ idx ] -> fieldname() ); + CSP_THROW( ValueError, + "csp Struct " << name << " attempted to add existing field " << m_fields[idx]->fieldname() ); } } -StructMeta::~StructMeta() -{ - m_default.reset(); -} +StructMeta::~StructMeta() { m_default.reset(); } Struct * StructMeta::createRaw() const { Struct * s; - //This weird looking new call actually is calling Struct::opereatore new( size_t, std::shared_ptr ) - //its not actually doing an in-place on StructMeta this - s = new ( shared_from_this() ) Struct( shared_from_this() ); + // This weird looking new call actually is calling Struct::opereatore new( size_t, std::shared_ptr ) + // its not actually doing an in-place on StructMeta this + s = new( shared_from_this() ) Struct( shared_from_this() ); initialize( s ); if( m_default ) - s -> deepcopyFrom( m_default.get() ); + s->deepcopyFrom( m_default.get() ); return s; } @@ -158,36 +162,74 @@ std::string StructMeta::layout() const for( auto & field : m_fields ) { char type = ' '; - switch( field -> type() -> type() ) + switch( field->type()->type() ) { - case CspType::Type::BOOL: type = 'b'; break; - case CspType::Type::INT8: type = 'c'; break; - case CspType::Type::UINT8: type = 'C'; break; - case CspType::Type::INT16: type = 'h'; break; - case CspType::Type::UINT16: type = 'H'; break; - case CspType::Type::INT32: type = 'd'; break; - case CspType::Type::UINT32: type = 'D'; break; - case CspType::Type::INT64: type = 'l'; break; - case CspType::Type::UINT64: type = 'L'; break; - case CspType::Type::DOUBLE: type = 'f'; break; - case CspType::Type::DATETIME: type = 't'; break; - case CspType::Type::TIMEDELTA: type = 'e'; break; - case CspType::Type::DATE: type = 'y'; break; - case CspType::Type::TIME: type = 'T'; break; - case CspType::Type::ENUM: type = 'N'; break; - case CspType::Type::STRUCT: type = 'S'; break; - case CspType::Type::STRING: type = 's'; break; - case CspType::Type::DIALECT_GENERIC: type = 'G'; break; - case CspType::Type::ARRAY: type = 'a'; break; + case CspType::Type::BOOL: + type = 'b'; + break; + case CspType::Type::INT8: + type = 'c'; + break; + case CspType::Type::UINT8: + type = 'C'; + break; + case CspType::Type::INT16: + type = 'h'; + break; + case CspType::Type::UINT16: + type = 'H'; + break; + case CspType::Type::INT32: + type = 'd'; + break; + case CspType::Type::UINT32: + type = 'D'; + break; + case CspType::Type::INT64: + type = 'l'; + break; + case CspType::Type::UINT64: + type = 'L'; + break; + case CspType::Type::DOUBLE: + type = 'f'; + break; + case CspType::Type::DATETIME: + type = 't'; + break; + case CspType::Type::TIMEDELTA: + type = 'e'; + break; + case CspType::Type::DATE: + type = 'y'; + break; + case CspType::Type::TIME: + type = 'T'; + break; + case CspType::Type::ENUM: + type = 'N'; + break; + case CspType::Type::STRUCT: + type = 'S'; + break; + case CspType::Type::STRING: + type = 's'; + break; + case CspType::Type::DIALECT_GENERIC: + type = 'G'; + break; + case CspType::Type::ARRAY: + type = 'a'; + break; case CspType::Type::UNKNOWN: case CspType::Type::NUM_TYPES: break; } - for( size_t c = 0; c < field -> size(); ++c ) - out[ field -> offset() + c ] = type; + for( size_t c = 0; c < field->size(); ++c ) + out[field->offset() + c] = type; - out[ field -> maskOffset() ] = 'M'; + out[field->maskOffset()] = 'M'; } return out; @@ -197,33 +239,33 @@ bool StructMeta::isDerivedType( const StructMeta * derived, const StructMeta * b { const StructMeta * b = derived; while( b && b != base ) - b = b -> m_base.get(); + b = b->m_base.get(); return b != nullptr; } void StructMeta::initialize( Struct * s ) const { - //TODO optimize initialize to use default if availbel instead of constructing - //and then replaceing with default + // TODO optimize initialize to use default if availbel instead of constructing + // and then replaceing with default if( isNative() ) { - memset( reinterpret_cast(s), 0, size() ); + memset( reinterpret_cast( s ), 0, size() ); return; } - memset( reinterpret_cast(s) + m_nativeStart, 0, partialNativeSize() ); - + memset( reinterpret_cast( s ) + m_nativeStart, 0, partialNativeSize() ); + if( !m_isPartialNative ) { for( size_t idx = m_firstPartialField; idx < m_firstNativePartialField; ++idx ) { - auto * field = m_fields[ idx ].get(); - static_cast( field ) -> initialize( s ); + auto * field = m_fields[idx].get(); + static_cast( field )->initialize( s ); } } - + if( m_base ) - m_base -> initialize( s ); + m_base->initialize( s ); } void StructMeta::copyFrom( const Struct * src, Struct * dest ) @@ -231,12 +273,13 @@ void StructMeta::copyFrom( const Struct * src, Struct * dest ) if( unlikely( src == dest ) ) return; - if( dest -> meta() != src -> meta() && - !StructMeta::isDerivedType( src -> meta(), dest -> meta() ) ) - CSP_THROW( TypeError, "Attempting to copy from struct type '" << src -> meta() -> name() << "' to struct type '" << dest -> meta() -> name() + if( dest->meta() != src->meta() && !StructMeta::isDerivedType( src->meta(), dest->meta() ) ) + CSP_THROW( TypeError, + "Attempting to copy from struct type '" + << src->meta()->name() << "' to struct type '" << dest->meta()->name() << "'. copy_from may only be used to copy from same type or derived types" ); - dest -> meta() -> copyFromImpl( src, dest, false ); + dest->meta()->copyFromImpl( src, dest, false ); } void StructMeta::deepcopyFrom( const Struct * src, Struct * dest ) @@ -244,45 +287,46 @@ void StructMeta::deepcopyFrom( const Struct * src, Struct * dest ) if( unlikely( src == dest ) ) return; - if( dest -> meta() != src -> meta() && - !StructMeta::isDerivedType( src -> meta(), dest -> meta() ) ) - CSP_THROW( TypeError, "Attempting to deepcopy from struct type '" << src -> meta() -> name() << "' to struct type '" << dest -> meta() -> name() + if( dest->meta() != src->meta() && !StructMeta::isDerivedType( src->meta(), dest->meta() ) ) + CSP_THROW( TypeError, + "Attempting to deepcopy from struct type '" + << src->meta()->name() << "' to struct type '" << dest->meta()->name() << "'. deepcopy_from may only be used to copy from same type or derived types" ); - dest -> meta() -> copyFromImpl( src, dest, true ); -} + dest->meta()->copyFromImpl( src, dest, true ); +} void StructMeta::copyFromImpl( const Struct * src, Struct * dest, bool deepcopy ) const { - //quick outs, if fully native we can memcpy the whole thing + // quick outs, if fully native we can memcpy the whole thing if( isNative() ) - memcpy( reinterpret_cast(dest), reinterpret_cast(src), size() ); + memcpy( reinterpret_cast( dest ), reinterpret_cast( src ), size() ); else { - //check if we have non-native types to handle here + // check if we have non-native types to handle here if( !m_isPartialNative ) { - //this logic relies on us allocating non-native fields up front + // this logic relies on us allocating non-native fields up front for( size_t idx = m_firstPartialField; idx < m_firstNativePartialField; ++idx ) { - auto * field = m_fields[ idx ].get(); + auto * field = m_fields[idx].get(); - if( field -> isSet( src ) ) + if( field->isSet( src ) ) if( deepcopy ) - static_cast( field ) -> deepcopyFrom( src, dest ); + static_cast( field )->deepcopyFrom( src, dest ); else - static_cast( field ) -> copyFrom( src, dest ); + static_cast( field )->copyFrom( src, dest ); else - static_cast( field ) -> clearValue( dest ); + static_cast( field )->clearValue( dest ); } } - //note that partialNative will include the mask bytes - this sets the native part and the mask - memcpy( reinterpret_cast(dest) + m_nativeStart, reinterpret_cast(src) + m_nativeStart, - partialNativeSize() ); - + // note that partialNative will include the mask bytes - this sets the native part and the mask + memcpy( reinterpret_cast( dest ) + m_nativeStart, + reinterpret_cast( src ) + m_nativeStart, partialNativeSize() ); + if( m_base ) - m_base -> copyFromImpl( src, dest, deepcopy ); + m_base->copyFromImpl( src, dest, deepcopy ); } } @@ -291,44 +335,44 @@ void StructMeta::updateFrom( const Struct * src, Struct * dest ) if( unlikely( src == dest ) ) return; - if( dest -> meta() != src -> meta() && - !StructMeta::isDerivedType( src -> meta(), dest -> meta() ) ) - CSP_THROW( TypeError, "Attempting to update from struct type '" << src -> meta() -> name() << "' to struct type '" << dest -> meta() -> name() + if( dest->meta() != src->meta() && !StructMeta::isDerivedType( src->meta(), dest->meta() ) ) + CSP_THROW( TypeError, + "Attempting to update from struct type '" + << src->meta()->name() << "' to struct type '" << dest->meta()->name() << "'. update_from may only be used to update from same type or derived types" ); - dest -> meta() -> updateFromImpl( src, dest ); -} + dest->meta()->updateFromImpl( src, dest ); +} void StructMeta::updateFromImpl( const Struct * src, Struct * dest ) const { for( Fields::const_iterator it = m_fields.begin() + m_firstPartialField; it != m_fields.end(); ++it ) { const StructFieldPtr & field = *it; - if( field -> isSet( src ) ) - field -> copyFrom( src, dest ); + if( field->isSet( src ) ) + field->copyFrom( src, dest ); } // update doesn't unset fields, so set the mask to the bitwise or of src and dest masks for( size_t i = m_maskLoc; i < m_maskLoc + m_maskSize; i++ ) { - std::byte * destMaskByte_ptr = reinterpret_cast(dest) + i; - const std::byte * srcMaskByte_ptr = reinterpret_cast(src) + i; - *destMaskByte_ptr = *destMaskByte_ptr | *srcMaskByte_ptr; + std::byte * destMaskByte_ptr = reinterpret_cast( dest ) + i; + const std::byte * srcMaskByte_ptr = reinterpret_cast( src ) + i; + *destMaskByte_ptr = *destMaskByte_ptr | *srcMaskByte_ptr; } if( m_base ) - m_base -> updateFromImpl( src, dest ); + m_base->updateFromImpl( src, dest ); } - bool StructMeta::isEqual( const Struct * x, const Struct * y ) const { - if( x -> meta() != y -> meta() ) + if( x->meta() != y->meta() ) return false; - //Note the curent use of memcpy for native types. This can cause issues on double comparisons - //esp if expecting NaN == NaN to be false, and when comparing -0.0 to +0.0.. may want to revisit - //We we do we may as well remove the basepadding copy + // Note the curent use of memcpy for native types. This can cause issues on double comparisons + // esp if expecting NaN == NaN to be false, and when comparing -0.0 to +0.0.. may want to revisit + // We we do we may as well remove the basepadding copy if( isNative() ) return memcmp( x, y, size() ) == 0; @@ -340,14 +384,14 @@ bool StructMeta::isEqual( const Struct * x, const Struct * y ) const { for( size_t idx = m_firstPartialField; idx < m_firstNativePartialField; ++idx ) { - auto * field = m_fields[ idx ].get(); - - if( field -> isSet( x ) != field -> isSet( y ) ) + auto * field = m_fields[idx].get(); + + if( field->isSet( x ) != field->isSet( y ) ) return false; - if( field -> isSet( x ) ) + if( field->isSet( x ) ) { - bool rv = static_cast( field ) -> isEqual( x, y ); + bool rv = static_cast( field )->isEqual( x, y ); if( !rv ) return false; } @@ -355,15 +399,19 @@ bool StructMeta::isEqual( const Struct * x, const Struct * y ) const } if( m_base ) - return m_base -> isEqual( x, y ); + return m_base->isEqual( x, y ); return true; } -template +template struct ScopedIncrement { - ScopedIncrement( T & v ) : m_v{ v } { ++m_v; } + ScopedIncrement( T & v ) + : m_v{ v } + { + ++m_v; + } ~ScopedIncrement() { --m_v; } private: @@ -373,7 +421,7 @@ struct ScopedIncrement size_t StructMeta::hash( const Struct * x ) const { - size_t hash = std::hash()( (uint64_t ) x -> meta() ); + size_t hash = std::hash()( (uint64_t)x->meta() ); if( isNative() ) return hash ^ csp::hash::hash_bytes( x, size() ); @@ -381,26 +429,27 @@ size_t StructMeta::hash( const Struct * x ) const static thread_local size_t s_recursionDepth = 0; ScopedIncrement guard( s_recursionDepth ); - if ( unlikely( s_recursionDepth > MAX_RECURSION_DEPTH ) ) + if( unlikely( s_recursionDepth > MAX_RECURSION_DEPTH ) ) CSP_THROW( RecursionError, - "Exceeded max recursion depth of " << MAX_RECURSION_DEPTH << " in " << name() << "::hash(), cannot hash cyclic data structure" ); + "Exceeded max recursion depth of " << MAX_RECURSION_DEPTH << " in " << name() + << "::hash(), cannot hash cyclic data structure" ); hash ^= csp::hash::hash_bytes( x + m_nativeStart, partialNativeSize() ); - + if( !m_isPartialNative ) { for( size_t idx = m_firstPartialField; idx < m_firstNativePartialField; ++idx ) { - auto * field = m_fields[ idx ].get(); - - //we dont incorporate unset fields, bitmask will cover them - if( field -> isSet( x ) ) - hash ^= static_cast( field ) -> hash( x ); + auto * field = m_fields[idx].get(); + + // we dont incorporate unset fields, bitmask will cover them + if( field->isSet( x ) ) + hash ^= static_cast( field )->hash( x ); } } - + if( m_base ) - hash ^= std::hash()( (uint64_t ) m_base.get() ) ^ m_base -> hash( x ); + hash ^= std::hash()( (uint64_t)m_base.get() ) ^ m_base->hash( x ); return hash; } @@ -409,30 +458,30 @@ void StructMeta::clear( Struct * s ) const { if( isNative() ) { - memset( reinterpret_cast(s), 0, size() ); + memset( reinterpret_cast( s ), 0, size() ); return; } - memset( reinterpret_cast(s) + m_nativeStart, 0, partialNativeSize() ); - + memset( reinterpret_cast( s ) + m_nativeStart, 0, partialNativeSize() ); + if( !m_isPartialNative ) { for( size_t idx = m_firstPartialField; idx < m_firstNativePartialField; ++idx ) { - auto * field = m_fields[ idx ].get(); - - if( field -> isSet( s ) ) - static_cast( field ) -> clearValue( s ); + auto * field = m_fields[idx].get(); + + if( field->isSet( s ) ) + static_cast( field )->clearValue( s ); } } - + if( m_base ) - m_base -> clear( s ); + m_base->clear( s ); } bool StructMeta::allFieldsSet( const Struct * s ) const { - size_t numLocalFields = m_fields.size() - m_firstPartialField; + size_t numLocalFields = m_fields.size() - m_firstPartialField; uint8_t numRemainingBits = numLocalFields % 8; const uint8_t * m = reinterpret_cast( s ) + m_maskLoc; @@ -450,7 +499,7 @@ bool StructMeta::allFieldsSet( const Struct * s ) const return false; } - return m_base ? m_base -> allFieldsSet( s ) : true; + return m_base ? m_base->allFieldsSet( s ) : true; } void StructMeta::destroy( Struct * s ) const @@ -462,34 +511,34 @@ void StructMeta::destroy( Struct * s ) const { for( size_t idx = m_firstPartialField; idx < m_firstNativePartialField; ++idx ) { - auto * field = m_fields[ idx ].get(); - static_cast( field ) -> destroy( s ); + auto * field = m_fields[idx].get(); + static_cast( field )->destroy( s ); } } - + if( m_base ) - m_base -> destroy( s ); + m_base->destroy( s ); } Struct::Struct( const std::shared_ptr & meta ) { - //Initialize meta shared_ptr + // Initialize meta shared_ptr new( hidden() ) HiddenData(); - hidden() -> refcount = 1; - hidden() -> meta = meta; - hidden() -> dialectPtr = nullptr; + hidden()->refcount = 1; + hidden()->meta = meta; + hidden()->dialectPtr = nullptr; } void * Struct::operator new( std::size_t count, const std::shared_ptr & meta ) { - //allocate meta -> size() ( data members ) + room for hidden data refcount and meta ptr - //Note that we currently assume sizeof( HiddenData ) is 16. We expect it to be a multiple of 8 - //so all further fields are proper aligned ( again assuming our max aligned field is 8 bytes ) - //so be careful changing this assert if for some reason in the future sizeof( HiddenData ) changes + // allocate meta -> size() ( data members ) + room for hidden data refcount and meta ptr + // Note that we currently assume sizeof( HiddenData ) is 16. We expect it to be a multiple of 8 + // so all further fields are proper aligned ( again assuming our max aligned field is 8 bytes ) + // so be careful changing this assert if for some reason in the future sizeof( HiddenData ) changes static_assert( sizeof( HiddenData ) % 8 == 0 ); - void * ptr = ::operator new( sizeof( HiddenData ) + meta -> size() ); + void * ptr = ::operator new( sizeof( HiddenData ) + meta->size() ); return reinterpret_cast( ptr ) + sizeof( HiddenData ); } @@ -499,4 +548,4 @@ void Struct::operator delete( void * ptr ) ::operator delete( p ); } -} +} // namespace csp diff --git a/cpp/csp/engine/TimeSeries.cpp b/cpp/csp/engine/TimeSeries.cpp index c05a10df..c503264a 100644 --- a/cpp/csp/engine/TimeSeries.cpp +++ b/cpp/csp/engine/TimeSeries.cpp @@ -1,12 +1,8 @@ #include -namespace csp { - using DuplicatePolicyEnum=TimeSeries::DuplicatePolicyEnum; - INIT_CSP_ENUM(DuplicatePolicyEnum, - "UNKNOWN", - "LAST_VALUE", - "FIRST_VALUE", - "ALL_VALUES", - ); +namespace csp +{ +using DuplicatePolicyEnum = TimeSeries::DuplicatePolicyEnum; +INIT_CSP_ENUM( DuplicatePolicyEnum, "UNKNOWN", "LAST_VALUE", "FIRST_VALUE", "ALL_VALUES", ); -} +} // namespace csp diff --git a/cpp/csp/engine/TimeSeriesProvider.cpp b/cpp/csp/engine/TimeSeriesProvider.cpp index 3428731e..4944b8ce 100644 --- a/cpp/csp/engine/TimeSeriesProvider.cpp +++ b/cpp/csp/engine/TimeSeriesProvider.cpp @@ -12,19 +12,17 @@ void TimeSeriesProvider::init( const CspTypePtr & type, Node * node ) m_node = node; m_type = type.get(); - switchCspType( type, [ this ]( auto tag ) - { - this -> initBuffer(); - } ); + switchCspType( type, [this]( auto tag ) { this->initBuffer(); } ); } void TimeSeriesProvider::reset() { - switchCspType( m_type, [ this ]( auto tag ) - { - using T = typename decltype(tag)::type; - static_cast *>( m_timeseries.get() ) -> reset(); - } ); + switchCspType( m_type, + [this]( auto tag ) + { + using T = typename decltype( tag )::type; + static_cast *>( m_timeseries.get() )->reset(); + } ); m_lastCycleCount = 0; m_propagator.clear(); @@ -40,9 +38,6 @@ bool TimeSeriesProvider::removeConsumer( Consumer * consumer, InputId id ) return m_propagator.removeConsumer( consumer, id ); } -const char * TimeSeriesProvider::name() const -{ - return m_node ? m_node -> name() : ""; -} +const char * TimeSeriesProvider::name() const { return m_node ? m_node->name() : ""; } -} +} // namespace csp diff --git a/cpp/csp/engine/VectorContainer.cpp b/cpp/csp/engine/VectorContainer.cpp index 3ab92a7c..7d0fecdb 100644 --- a/cpp/csp/engine/VectorContainer.cpp +++ b/cpp/csp/engine/VectorContainer.cpp @@ -1,45 +1,45 @@ -#include #include #include +#include #include namespace { -template< typename T > +template class TypedVectorContainer : public csp::VectorContainer { protected: - void *getVectorUntyped() - { - return &m_vector; - } + void * getVectorUntyped() { return &m_vector; } private: std::vector m_vector; }; -} +} // namespace namespace csp { -std::unique_ptr VectorContainer::createForCspType(CspTypePtr &type, bool optionalValues) +std::unique_ptr VectorContainer::createForCspType( CspTypePtr & type, bool optionalValues ) { - return createForCspType(type.get()); + return createForCspType( type.get() ); } -std::unique_ptr VectorContainer::createForCspType( const CspType *type, bool optionalValues ) +std::unique_ptr VectorContainer::createForCspType( const CspType * type, bool optionalValues ) { - return AllCspTypeSwitch::invoke(type, [optionalValues]( auto tag ) - { - if(optionalValues) + return AllCspTypeSwitch::invoke( + type, + [optionalValues]( auto tag ) { - return std::unique_ptr( new TypedVectorContainer> ); - } - else - { - return std::unique_ptr( new TypedVectorContainer ); - } - }); + if( optionalValues ) + { + return std::unique_ptr( + new TypedVectorContainer> ); + } + else + { + return std::unique_ptr( new TypedVectorContainer ); + } + } ); } -} +} // namespace csp diff --git a/cpp/csp/python/Conversions.cpp b/cpp/csp/python/Conversions.cpp index af66c8ff..649ffa2d 100644 --- a/cpp/csp/python/Conversions.cpp +++ b/cpp/csp/python/Conversions.cpp @@ -6,20 +6,16 @@ namespace csp::python PyObject * lastValueToPython( const csp::TimeSeriesProvider * ts ) { - return switchCspType( ts -> type(), - [ ts ]( auto tag ) - { - return toPython( ts -> lastValueTyped(), *ts -> type() ); - } ); + return switchCspType( ts->type(), [ts]( auto tag ) + { return toPython( ts->lastValueTyped(), *ts->type() ); } ); } -PyObject * valueAtIndexToPython( const csp::TimeSeriesProvider * ts, int32_t index) +PyObject * valueAtIndexToPython( const csp::TimeSeriesProvider * ts, int32_t index ) { - return switchCspType( ts -> type(), - [ ts, index ]( auto tag ) - { - return toPython( ts -> valueAtIndex( index ), *ts -> type() ); + return switchCspType( ts->type(), + [ts, index]( auto tag ) { + return toPython( ts->valueAtIndex( index ), *ts->type() ); } ); } -} +} // namespace csp::python diff --git a/cpp/csp/python/CspTypeFactory.cpp b/cpp/csp/python/CspTypeFactory.cpp index a1bfd027..136861da 100644 --- a/cpp/csp/python/CspTypeFactory.cpp +++ b/cpp/csp/python/CspTypeFactory.cpp @@ -20,17 +20,19 @@ CspTypePtr & CspTypeFactory::typeFromPyType( PyObject * pyTypeObj ) // List objects shouldn't be cached since they are temporary objects if( PyList_Check( pyTypeObj ) ) { - if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 1 && PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 2 ) - CSP_THROW( TypeError, "Expected list types post-normalization to be one or two elements: sub-type and optional FastList flag" ); + if( PyList_GET_SIZE( (PyObject *)pyTypeObj ) != 1 && PyList_GET_SIZE( (PyObject *)pyTypeObj ) != 2 ) + CSP_THROW( TypeError, + "Expected list types post-normalization to be one or two elements: sub-type and optional " + "FastList flag" ); - PyObject *pySubType = PyList_GET_ITEM( pyTypeObj, 0 ); + PyObject * pySubType = PyList_GET_ITEM( pyTypeObj, 0 ); if( !PyType_Check( pySubType ) ) CSP_THROW( TypeError, "nested typed lists are not supported" ); bool useFastList = false; - if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) == 2 ) + if( PyList_GET_SIZE( (PyObject *)pyTypeObj ) == 2 ) { - PyObject *pyUseFastList = PyList_GET_ITEM( pyTypeObj, 1 ); + PyObject * pyUseFastList = PyList_GET_ITEM( pyTypeObj, 1 ); if( !PyBool_Check( pyUseFastList ) || pyUseFastList != Py_True ) CSP_THROW( TypeError, "expected bool True as second list type argument" ); useFastList = true; @@ -40,52 +42,50 @@ CspTypePtr & CspTypeFactory::typeFromPyType( PyObject * pyTypeObj ) return CspArrayType::create( elemType, useFastList ); } - PyTypeObject *pyType = (PyTypeObject*) pyTypeObj; - auto rv = m_cache.emplace( pyType, nullptr ); + PyTypeObject * pyType = (PyTypeObject *)pyTypeObj; + auto rv = m_cache.emplace( pyType, nullptr ); if( rv.second ) { if( pyType == &PyFloat_Type ) - rv.first -> second = csp::CspType::DOUBLE(); + rv.first->second = csp::CspType::DOUBLE(); else if( pyType == &PyLong_Type ) - rv.first -> second = csp::CspType::INT64(); + rv.first->second = csp::CspType::INT64(); else if( pyType == &PyBool_Type ) - rv.first -> second = csp::CspType::BOOL(); + rv.first->second = csp::CspType::BOOL(); else if( pyType == &PyUnicode_Type ) - rv.first -> second = csp::CspType::STRING(); + rv.first->second = csp::CspType::STRING(); else if( pyType == &PyBytes_Type ) - rv.first -> second = csp::CspType::BYTES(); + rv.first->second = csp::CspType::BYTES(); else if( PyType_IsSubtype( pyType, &PyStruct::PyType ) ) { - auto meta = ( ( PyStructMeta * ) pyType ) -> structMeta; - rv.first -> second = std::make_shared( meta ); + auto meta = ( (PyStructMeta *)pyType )->structMeta; + rv.first->second = std::make_shared( meta ); } else if( PyType_IsSubtype( pyType, &PyCspEnum::PyType ) ) { - auto meta = ( ( PyCspEnumMeta * ) pyType ) -> enumMeta; - rv.first -> second = std::make_shared( meta ); + auto meta = ( (PyCspEnumMeta *)pyType )->enumMeta; + rv.first->second = std::make_shared( meta ); } - else if( pyType == PyDateTimeAPI -> DateTimeType ) - rv.first -> second = csp::CspType::DATETIME(); - else if( pyType == PyDateTimeAPI -> DeltaType ) - rv.first -> second = csp::CspType::TIMEDELTA(); - else if( pyType == PyDateTimeAPI -> DateType ) - rv.first -> second = csp::CspType::DATE(); - else if( pyType == PyDateTimeAPI -> TimeType ) - rv.first -> second = csp::CspType::TIME(); + else if( pyType == PyDateTimeAPI->DateTimeType ) + rv.first->second = csp::CspType::DATETIME(); + else if( pyType == PyDateTimeAPI->DeltaType ) + rv.first->second = csp::CspType::TIMEDELTA(); + else if( pyType == PyDateTimeAPI->DateType ) + rv.first->second = csp::CspType::DATE(); + else if( pyType == PyDateTimeAPI->TimeType ) + rv.first->second = csp::CspType::TIME(); else { if( !PyType_Check( pyType ) ) - CSP_THROW( TypeError, "expected python type for CspType got " << PyObjectPtr::incref( ( PyObject * ) pyType ) ); - rv.first -> second = csp::CspType::DIALECT_GENERIC(); + CSP_THROW( TypeError, + "expected python type for CspType got " << PyObjectPtr::incref( (PyObject *)pyType ) ); + rv.first->second = csp::CspType::DIALECT_GENERIC(); } } - return rv.first -> second; -} - -void CspTypeFactory::removeCachedType( PyTypeObject * pyType ) -{ - m_cache.erase( pyType ); + return rv.first->second; } -} +void CspTypeFactory::removeCachedType( PyTypeObject * pyType ) { m_cache.erase( pyType ); } + +} // namespace csp::python diff --git a/cpp/csp/python/NumpyConversions.cpp b/cpp/csp/python/NumpyConversions.cpp index 98254403..9a758c33 100644 --- a/cpp/csp/python/NumpyConversions.cpp +++ b/cpp/csp/python/NumpyConversions.cpp @@ -1,33 +1,31 @@ -//this is included first so that we do include without NO_IMPORT_ARRAY defined, which is done in NumpyConversions.h +// this is included first so that we do include without NO_IMPORT_ARRAY defined, which is done in NumpyConversions.h #include - #include #include -#include #include +#include namespace csp::python { static bool numpy_initialized = false; -PyObject * valuesAtIndexToNumpy( ValueType valueType, const csp::TimeSeriesProvider * ts, int32_t startIndex, int32_t endIndex, - autogen::TimeIndexPolicy startPolicy, autogen::TimeIndexPolicy endPolicy, - DateTime startDt, DateTime endDt ) +PyObject * valuesAtIndexToNumpy( ValueType valueType, const csp::TimeSeriesProvider * ts, int32_t startIndex, + int32_t endIndex, autogen::TimeIndexPolicy startPolicy, + autogen::TimeIndexPolicy endPolicy, DateTime startDt, DateTime endDt ) { if( !numpy_initialized ) { - import_array() - numpy_initialized = true; + import_array() numpy_initialized = true; } - return switchCspType( ts -> type(), - [ valueType, ts, startIndex, endIndex, startPolicy, endPolicy, startDt, endDt ]( auto tag ) + return switchCspType( ts->type(), + [valueType, ts, startIndex, endIndex, startPolicy, endPolicy, startDt, endDt]( auto tag ) { - return createNumpyArray( valueType, ts, startIndex, endIndex, startPolicy, - endPolicy, startDt, endDt ); + return createNumpyArray( + valueType, ts, startIndex, endIndex, startPolicy, endPolicy, startDt, endDt ); } ); } @@ -38,43 +36,44 @@ int64_t scalingFromNumpyDtUnit( NPY_DATETIMEUNIT base ) case NPY_FR_ns: return 1; case NPY_FR_us: - return csp::TimeDelta::fromMicroseconds(1).asNanoseconds(); + return csp::TimeDelta::fromMicroseconds( 1 ).asNanoseconds(); case NPY_FR_ms: - return csp::TimeDelta::fromMilliseconds(1).asNanoseconds(); + return csp::TimeDelta::fromMilliseconds( 1 ).asNanoseconds(); case NPY_FR_s: - return csp::TimeDelta::fromSeconds(1).asNanoseconds(); + return csp::TimeDelta::fromSeconds( 1 ).asNanoseconds(); case NPY_FR_m: - return csp::TimeDelta::fromMinutes(1).asNanoseconds(); + return csp::TimeDelta::fromMinutes( 1 ).asNanoseconds(); case NPY_FR_h: - return csp::TimeDelta::fromHours(1).asNanoseconds(); + return csp::TimeDelta::fromHours( 1 ).asNanoseconds(); case NPY_FR_D: - return csp::TimeDelta::fromDays(1).asNanoseconds(); + return csp::TimeDelta::fromDays( 1 ).asNanoseconds(); case NPY_FR_W: - return csp::TimeDelta::fromDays(7).asNanoseconds(); + return csp::TimeDelta::fromDays( 7 ).asNanoseconds(); default: - CSP_THROW(csp::NotImplemented, "datetime resolution not supported or invalid - saw NPY_DATETIMEUNIT value " << base ); - return 0; // never reached, but keeps compiler happy + CSP_THROW( csp::NotImplemented, + "datetime resolution not supported or invalid - saw NPY_DATETIMEUNIT value " << base ); + return 0; // never reached, but keeps compiler happy } } -NPY_DATETIMEUNIT datetimeUnitFromDescr( PyArray_Descr* descr ) +NPY_DATETIMEUNIT datetimeUnitFromDescr( PyArray_Descr * descr ) { - PyArray_DatetimeDTypeMetaData* dtypeMeta = (PyArray_DatetimeDTypeMetaData*)(descr -> c_metadata); - PyArray_DatetimeMetaData* dtMeta = &(dtypeMeta -> meta); - return dtMeta -> base; + PyArray_DatetimeDTypeMetaData * dtypeMeta = (PyArray_DatetimeDTypeMetaData *)( descr->c_metadata ); + PyArray_DatetimeMetaData * dtMeta = &( dtypeMeta->meta ); + return dtMeta->base; } static std::wstring_convert, char32_t> wstr_converter; -void stringFromNumpyStr( void* data, std::string& out, char numpy_type, int elem_size_bytes ) +void stringFromNumpyStr( void * data, std::string & out, char numpy_type, int elem_size_bytes ) { - // strings from numpy arrays are fixed width and zero filled. + // strings from numpy arrays are fixed width and zero filled. // if the last char is 0, can treat as null terminated, else use full width - if( numpy_type == NPY_UNICODELTR) + if( numpy_type == NPY_UNICODELTR ) { - const char32_t* const raw_value = (const char32_t *) data; - const int field_size = elem_size_bytes / sizeof( char32_t ); + const char32_t * const raw_value = (const char32_t *)data; + const int field_size = elem_size_bytes / sizeof( char32_t ); if( raw_value[field_size - 1] == 0 ) { @@ -89,7 +88,7 @@ void stringFromNumpyStr( void* data, std::string& out, char numpy_type, int elem } else if( numpy_type == NPY_STRINGLTR || numpy_type == NPY_STRINGLTR2 ) { - const char * const raw_value = (const char *) data; + const char * const raw_value = (const char *)data; if( raw_value[elem_size_bytes - 1] == 0 ) out = std::string( raw_value ); @@ -98,14 +97,14 @@ void stringFromNumpyStr( void* data, std::string& out, char numpy_type, int elem } else if( numpy_type == NPY_CHARLTR ) { - const char * const raw_value = (const char *) data; - out = std::string( raw_value, 1 ); + const char * const raw_value = (const char *)data; + out = std::string( raw_value, 1 ); } } void validateNumpyTypeVsCspType( const CspTypePtr & type, char numpy_type_char ) { - csp::CspType::Type cspType = type -> type(); + csp::CspType::Type cspType = type->type(); switch( numpy_type_char ) { @@ -126,7 +125,9 @@ void validateNumpyTypeVsCspType( const CspTypePtr & type, char numpy_type_char ) case NPY_ULONGLTR: case NPY_LONGLONGLTR: case NPY_ULONGLONGLTR: - CSP_THROW( ValueError, "numpy type " << numpy_type_char << " (int type that can't cleanly convert to long) not supported" ); + CSP_THROW( ValueError, + "numpy type " << numpy_type_char + << " (int type that can't cleanly convert to long) not supported" ); case NPY_HALFLTR: CSP_THROW( ValueError, "numpy type " << numpy_type_char << " (numpy half float) not supported" ); case NPY_FLOATLTR: @@ -165,5 +166,4 @@ void validateNumpyTypeVsCspType( const CspTypePtr & type, char numpy_type_char ) } } - -} +} // namespace csp::python diff --git a/cpp/csp/python/PyAdapterManager.cpp b/cpp/csp/python/PyAdapterManager.cpp index 3a9ec721..9544c0bb 100644 --- a/cpp/csp/python/PyAdapterManager.cpp +++ b/cpp/csp/python/PyAdapterManager.cpp @@ -1,9 +1,9 @@ #include #include -#include -#include #include #include +#include +#include #include #include @@ -13,19 +13,17 @@ namespace csp::python class PyAdapterManager : public AdapterManager { public: - PyAdapterManager( Engine * engine, PyObjectPtr self ) : AdapterManager( engine ), - m_self( self ) + PyAdapterManager( Engine * engine, PyObjectPtr self ) + : AdapterManager( engine ) + , m_self( self ) { } - const char * name() const override - { - return Py_TYPE( m_self.ptr() ) -> tp_name; - } + const char * name() const override { return Py_TYPE( m_self.ptr() )->tp_name; } void start( DateTime start, DateTime end ) override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_self.ptr(), "start", "OO", + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_self.ptr(), "start", "OO", PyObjectPtr::own( toPython( start ) ).ptr(), PyObjectPtr::own( toPython( end ) ).ptr() ) ); if( !rv.ptr() ) @@ -36,7 +34,7 @@ class PyAdapterManager : public AdapterManager void stop() override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_self.ptr(), "stop", nullptr ) ); + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_self.ptr(), "stop", nullptr ) ); if( !rv.ptr() ) CSP_THROW( PythonPassthrough, "" ); @@ -44,20 +42,20 @@ class PyAdapterManager : public AdapterManager DateTime processNextSimTimeSlice( DateTime time ) override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_self.ptr(), "process_next_sim_timeslice", "O", + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_self.ptr(), "process_next_sim_timeslice", "O", PyObjectPtr::own( toPython( time ) ).ptr() ) ); if( !rv.ptr() ) { if( PyErr_Occurred() == PyExc_KeyboardInterrupt ) { - rootEngine() -> shutdown(); + rootEngine()->shutdown(); return DateTime::NONE(); } CSP_THROW( PythonPassthrough, "" ); } - + return rv.ptr() == Py_None ? DateTime::NONE() : fromPython( rv.ptr() ); } @@ -65,80 +63,86 @@ class PyAdapterManager : public AdapterManager PyObjectPtr m_self; }; -static PyObject * PyAdapterManager_PyObject_starttime( PyAdapterManager_PyObject * self ) { return toPython( self -> manager -> starttime() ); } -static PyObject * PyAdapterManager_PyObject_endtime( PyAdapterManager_PyObject * self ) { return toPython( self -> manager -> endtime() ); } +static PyObject * PyAdapterManager_PyObject_starttime( PyAdapterManager_PyObject * self ) +{ + return toPython( self->manager->starttime() ); +} +static PyObject * PyAdapterManager_PyObject_endtime( PyAdapterManager_PyObject * self ) +{ + return toPython( self->manager->endtime() ); +} -static int PyAdapterManager_init( PyAdapterManager_PyObject *self, PyObject *args, PyObject *kwds ) +static int PyAdapterManager_init( PyAdapterManager_PyObject * self, PyObject * args, PyObject * kwds ) { CSP_BEGIN_METHOD; PyEngine * pyEngine; - - if( !PyArg_ParseTuple( args, "O!", - &PyEngine::PyType, &pyEngine ) ) + + if( !PyArg_ParseTuple( args, "O!", &PyEngine::PyType, &pyEngine ) ) CSP_THROW( PythonPassthrough, "" ); - self -> manager = pyEngine -> engine() -> createOwnedObject( PyObjectPtr::incref( ( PyObject * ) self ) ); + self->manager = pyEngine->engine()->createOwnedObject( PyObjectPtr::incref( (PyObject *)self ) ); CSP_RETURN_INT; } -static PyMethodDef PyAdapterManager_methods[] = { - { "starttime", (PyCFunction) PyAdapterManager_PyObject_starttime, METH_NOARGS, "starttime" }, - { "endtime", (PyCFunction) PyAdapterManager_PyObject_endtime, METH_NOARGS, "endtime" }, - {NULL} -}; +static PyMethodDef PyAdapterManager_methods[] + = { { "starttime", (PyCFunction)PyAdapterManager_PyObject_starttime, METH_NOARGS, "starttime" }, + { "endtime", (PyCFunction)PyAdapterManager_PyObject_endtime, METH_NOARGS, "endtime" }, + { NULL } }; PyTypeObject PyAdapterManager_PyObject::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyAdapterManager", /* tp_name */ - sizeof(PyAdapterManager_PyObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - "csp adapter manager", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyAdapterManager_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - ( initproc ) PyAdapterManager_init, /* tp_init */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyAdapterManager", /* tp_name */ + sizeof( PyAdapterManager_PyObject ), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + "csp adapter manager", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyAdapterManager_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)PyAdapterManager_init, /* tp_init */ 0, PyType_GenericNew, - 0, /* tp_free */ /* Low-level free-memory routine */ - 0, /* tp_is_gc */ /* For PyObject_IS_GC */ - 0, /* tp_bases */ - 0, /* tp_mro */ /* method resolution order */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0 /* tp_finalize */ + 0, + /* tp_free */ /* Low-level free-memory routine */ + 0, + /* tp_is_gc */ /* For PyObject_IS_GC */ + 0, /* tp_bases */ + 0, + /* tp_mro */ /* method resolution order */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0 /* tp_finalize */ }; REGISTER_TYPE_INIT( &PyAdapterManager_PyObject::PyType, "PyAdapterManager" ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyAdapterManagerWrapper.cpp b/cpp/csp/python/PyAdapterManagerWrapper.cpp index 36329165..e401081f 100644 --- a/cpp/csp/python/PyAdapterManagerWrapper.cpp +++ b/cpp/csp/python/PyAdapterManagerWrapper.cpp @@ -2,8 +2,8 @@ #include #include #include -#include #include +#include #include namespace csp::python @@ -16,9 +16,7 @@ PyObject * PyAdapterManagerWrapper::create( Creator creator, PyObject * args ) PyEngine * pyEngine = nullptr; PyObject * pyProperties = nullptr; - if( !PyArg_ParseTuple( args, "O!O!", - &PyEngine::PyType, &pyEngine, - &PyDict_Type, &pyProperties ) ) + if( !PyArg_ParseTuple( args, "O!O!", &PyEngine::PyType, &pyEngine, &PyDict_Type, &pyProperties ) ) CSP_THROW( PythonPassthrough, "" ); auto * adapterMgr = creator( pyEngine, fromPython( pyProperties ) ); @@ -29,15 +27,16 @@ PyObject * PyAdapterManagerWrapper::create( Creator creator, PyObject * args ) csp::AdapterManager * PyAdapterManagerWrapper::extractAdapterManager( PyObject * wrapper ) { - return ( csp::AdapterManager * ) PyCapsule_GetPointer( wrapper, "adapterMgr" ); + return (csp::AdapterManager *)PyCapsule_GetPointer( wrapper, "adapterMgr" ); } -static StatusAdapter * create_status_adapter( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, PushMode pushMode, PyObject * args ) +static StatusAdapter * create_status_adapter( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, + PushMode pushMode, PyObject * args ) { auto & cspType = pyTypeAsCspType( pyType ); - return manager -> createStatusAdapter( cspType, pushMode ); + return manager->createStatusAdapter( cspType, pushMode ); } REGISTER_INPUT_ADAPTER( _status_adapter, create_status_adapter ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyBasketInputProxy.cpp b/cpp/csp/python/PyBasketInputProxy.cpp index fa3b4bf4..d9ab1bed 100644 --- a/cpp/csp/python/PyBasketInputProxy.cpp +++ b/cpp/csp/python/PyBasketInputProxy.cpp @@ -12,10 +12,13 @@ namespace csp::python { -template +template struct TsIterator { - TsIterator( ITER i ) : it( i ) {} + TsIterator( ITER i ) + : it( i ) + { + } PyObject * iternext() { @@ -34,155 +37,153 @@ struct TsIterator CSP_RETURN_NULL; } - ITER it; + ITER it; GETTER getter; }; -struct ValueGetter +struct ValueGetter { - template PyObject * operator()( const IterT & iter ) { return lastValueToPython( iter.get() ); } + template + PyObject * operator()( const IterT & iter ) + { + return lastValueToPython( iter.get() ); + } }; template -struct KeyGetter +struct KeyGetter { PyBasketInputProxyT * proxy; - template PyObject * operator()( const IterT & iter ) { return toPython( proxy -> key( iter.elemId() ) ); } + template + PyObject * operator()( const IterT & iter ) + { + return toPython( proxy->key( iter.elemId() ) ); + } }; template -struct ItemGetter +struct ItemGetter { PyBasketInputProxyT * proxy; - template PyObject * operator()( const IterT & iter ) - { + template + PyObject * operator()( const IterT & iter ) + { PyObject * rv = PyTuple_New( 2 ); if( !rv ) CSP_THROW( PythonPassthrough, "" ); - PyTuple_SET_ITEM( rv, 0, toPython( proxy -> key( iter.elemId() ) ) ); + PyTuple_SET_ITEM( rv, 0, toPython( proxy->key( iter.elemId() ) ) ); PyTuple_SET_ITEM( rv, 1, lastValueToPython( iter.get() ) ); return rv; } }; -using ValidValuesIterator = TsIterator; -template using ValidKeysIterator = TsIterator>; -template using ValidItemsIterator = TsIterator>; +using ValidValuesIterator = TsIterator; +template +using ValidKeysIterator = TsIterator>; +template +using ValidItemsIterator = TsIterator>; -using TickedValuesIterator = TsIterator; -template using TickedKeysIterator = TsIterator>; -template using TickedItemsIterator = TsIterator>; +using TickedValuesIterator = TsIterator; +template +using TickedKeysIterator = TsIterator>; +template +using TickedItemsIterator = TsIterator>; -PyBaseBasketInputProxy::PyBaseBasketInputProxy( PyNode * node, INOUT_ID_TYPE id ) : m_node( node ), - m_id( id ) +PyBaseBasketInputProxy::PyBaseBasketInputProxy( PyNode * node, INOUT_ID_TYPE id ) + : m_node( node ) + , m_id( id ) { } -bool PyBaseBasketInputProxy::ticked() const -{ - return basketInfo() -> ticked(); -} +bool PyBaseBasketInputProxy::ticked() const { return basketInfo()->ticked(); } -bool PyBaseBasketInputProxy::valid() const -{ - return basketInfo() -> allValid(); -} +bool PyBaseBasketInputProxy::valid() const { return basketInfo()->allValid(); } -InputBasketInfo * PyBaseBasketInputProxy::basketInfo() -{ - return m_node -> inputBasket( m_id ); -} +InputBasketInfo * PyBaseBasketInputProxy::basketInfo() { return m_node->inputBasket( m_id ); } -bool PyBaseBasketInputProxy::makeActive() -{ - return m_node -> makeBasketActive( m_id ); -} +bool PyBaseBasketInputProxy::makeActive() { return m_node->makeBasketActive( m_id ); } -bool PyBaseBasketInputProxy::makePassive() -{ - return m_node -> makeBasketPassive( m_id ); -} +bool PyBaseBasketInputProxy::makePassive() { return m_node->makeBasketPassive( m_id ); } void PyBaseBasketInputProxy::setBufferingPolicy( int32_t tickCount, TimeDelta tickHistory ) { auto * basket = basketInfo(); - for( size_t elemId = 0; elemId < basket -> size(); ++elemId ) + for( size_t elemId = 0; elemId < basket->size(); ++elemId ) { - auto * ts = const_cast( m_node -> input( InputId( m_id, elemId ) ) ); + auto * ts = const_cast( m_node->input( InputId( m_id, elemId ) ) ); if( tickCount > 0 ) - ts -> setTickCountPolicy( tickCount ); - + ts->setTickCountPolicy( tickCount ); + if( !tickHistory.isNone() && tickHistory > TimeDelta::ZERO() ) - ts -> setTickTimeWindowPolicy( tickHistory ); + ts->setTickTimeWindowPolicy( tickHistory ); } - //if its dynamic, set the policy on the basket as well so that newly added keys get the same policy applied - if( basket -> isDynamicBasket() ) + // if its dynamic, set the policy on the basket as well so that newly added keys get the same policy applied + if( basket->isDynamicBasket() ) { - auto * dynBasket = static_cast( basket ); + auto * dynBasket = static_cast( basket ); if( tickCount > 0 ) - dynBasket -> setTickCountPolicy( tickCount ); + dynBasket->setTickCountPolicy( tickCount ); if( !tickHistory.isNone() && tickHistory > TimeDelta::ZERO() ) - dynBasket -> setTickTimeWindowPolicy( tickHistory ); + dynBasket->setTickTimeWindowPolicy( tickHistory ); } } -//reactive methods +// reactive methods static PyObject * PyBaseBasketInputProxy_ticked( PyBaseBasketInputProxy * proxy ) { CSP_BEGIN_METHOD; - return toPython( proxy -> ticked() ); + return toPython( proxy->ticked() ); CSP_RETURN_NONE; } static PyObject * PyBaseBasketInputProxy_valid( PyBaseBasketInputProxy * proxy ) { CSP_BEGIN_METHOD; - return toPython( proxy -> valid() ); + return toPython( proxy->valid() ); CSP_RETURN_NONE; } static PyObject * PyBaseBasketInputProxy_make_active( PyBaseBasketInputProxy * proxy, PyObject * args ) { CSP_BEGIN_METHOD; - return toPython( proxy -> makeActive() ); + return toPython( proxy->makeActive() ); CSP_RETURN_NONE; } static PyObject * PyBaseBasketInputProxy_make_passive( PyBaseBasketInputProxy * proxy, PyObject * args ) { CSP_BEGIN_METHOD; - return toPython( proxy -> makePassive() ); + return toPython( proxy->makePassive() ); CSP_RETURN_NONE; } static PyObject * PyBaseBasketInputProxy_validvalues( PyBaseBasketInputProxy * proxy ) { CSP_BEGIN_METHOD; - ValidValuesIterator iter( proxy -> basketInfo() -> begin_valid() ); + ValidValuesIterator iter( proxy->basketInfo()->begin_valid() ); return PyIterator::create( iter ); CSP_RETURN_NONE; } -static PyObject * PyBaseBasketInputProxy_set_buffering_policy( PyBaseBasketInputProxy * proxy, PyObject * args, PyObject * kwargs ) +static PyObject * PyBaseBasketInputProxy_set_buffering_policy( PyBaseBasketInputProxy * proxy, PyObject * args, + PyObject * kwargs ) { CSP_BEGIN_METHOD; PyObject * tickCount = nullptr; PyObject * tickHistory = nullptr; static const char * kwlist[] = { "tick_count", "tick_history", nullptr }; - if( !PyArg_ParseTupleAndKeywords( args, kwargs, "|O!O", ( char ** ) kwlist, - &PyLong_Type, &tickCount, - &tickHistory ) ) + if( !PyArg_ParseTupleAndKeywords( args, kwargs, "|O!O", (char **)kwlist, &PyLong_Type, &tickCount, &tickHistory ) ) CSP_THROW( PythonPassthrough, "" ); if( !tickCount && !tickHistory ) CSP_THROW( TypeError, "csp.set_buffering_policy expected at least one of tick_count or tick_history" ); - proxy -> setBufferingPolicy( tickCount ? fromPython( tickCount ) : -1, - tickHistory ? fromPython( tickHistory ) : TimeDelta::NONE() ); + proxy->setBufferingPolicy( tickCount ? fromPython( tickCount ) : -1, + tickHistory ? fromPython( tickHistory ) : TimeDelta::NONE() ); CSP_RETURN_NONE; } @@ -190,7 +191,7 @@ template static PyObject * PyBaseBasketInputProxy_validitems( PyBasketInputProxyT * proxy ) { CSP_BEGIN_METHOD; - ValidItemsIterator iter( proxy -> basketInfo() -> begin_valid() ); + ValidItemsIterator iter( proxy->basketInfo()->begin_valid() ); iter.getter.proxy = proxy; return PyIterator>::create( iter ); CSP_RETURN_NONE; @@ -200,7 +201,7 @@ template static PyObject * PyBaseBasketInputProxy_validkeys( PyBasketInputProxyT * proxy ) { CSP_BEGIN_METHOD; - ValidKeysIterator iter( proxy -> basketInfo() -> begin_valid() ); + ValidKeysIterator iter( proxy->basketInfo()->begin_valid() ); iter.getter.proxy = proxy; return PyIterator>::create( iter ); CSP_RETURN_NONE; @@ -209,8 +210,8 @@ static PyObject * PyBaseBasketInputProxy_validkeys( PyBasketInputProxyT * proxy static PyObject * PyBaseBasketInputProxy_tickedvalues( PyBaseBasketInputProxy * proxy ) { CSP_BEGIN_METHOD; - TickedValuesIterator iter( proxy -> basketInfo() -> begin_ticked() ); - return PyIterator::create( iter ); + TickedValuesIterator iter( proxy->basketInfo()->begin_ticked() ); + return PyIterator::create( iter ); CSP_RETURN_NONE; } @@ -218,7 +219,7 @@ template static PyObject * PyBaseBasketInputProxy_tickeditems( PyBasketInputProxyT * proxy ) { CSP_BEGIN_METHOD; - TickedItemsIterator iter( proxy -> basketInfo() -> begin_ticked() ); + TickedItemsIterator iter( proxy->basketInfo()->begin_ticked() ); iter.getter.proxy = proxy; return PyIterator>::create( iter ); CSP_RETURN_NONE; @@ -228,24 +229,22 @@ template static PyObject * PyBaseBasketInputProxy_tickedkeys( PyBasketInputProxyT * proxy ) { CSP_BEGIN_METHOD; - TickedKeysIterator iter( proxy -> basketInfo() -> begin_ticked() ); + TickedKeysIterator iter( proxy->basketInfo()->begin_ticked() ); iter.getter.proxy = proxy; return PyIterator>::create( iter ); CSP_RETURN_NONE; } -static Py_ssize_t PyBaseBasketInputProxy_len( PyBaseBasketInputProxy * proxy ) +static Py_ssize_t PyBaseBasketInputProxy_len( PyBaseBasketInputProxy * proxy ) { CSP_BEGIN_METHOD; - return proxy -> basketInfo() -> size(); + return proxy->basketInfo()->size(); CSP_RETURN_INT; } - - -//ListBasket specific methods -PyListBasketInputProxy::PyListBasketInputProxy( PyNode * node, INOUT_ID_TYPE id, - size_t shape ) : PyBaseBasketInputProxy( node, id ) +// ListBasket specific methods +PyListBasketInputProxy::PyListBasketInputProxy( PyNode * node, INOUT_ID_TYPE id, size_t shape ) + : PyBaseBasketInputProxy( node, id ) { for( size_t elemId = 0; elemId < shape; ++elemId ) m_proxies.emplace_back( PyInputProxyPtr::own( PyInputProxy::create( node, InputId( id, elemId ) ) ) ); @@ -254,9 +253,11 @@ PyListBasketInputProxy::PyListBasketInputProxy( PyNode * node, INOUT_ID_TYPE id, PyListBasketInputProxy * PyListBasketInputProxy::create( PyNode * node, INOUT_ID_TYPE id, size_t shape ) { if( shape > InputId::maxBasketElements() ) - CSP_THROW( ValueError, "List basket size of " << shape << " exceeds basket size limit of " << InputId::maxBasketElements() << " in node " << node -> name() ); + CSP_THROW( ValueError, + "List basket size of " << shape << " exceeds basket size limit of " << InputId::maxBasketElements() + << " in node " << node->name() ); - PyListBasketInputProxy * proxy = ( PyListBasketInputProxy * ) PyType.tp_new( &PyType, nullptr, nullptr ); + PyListBasketInputProxy * proxy = (PyListBasketInputProxy *)PyType.tp_new( &PyType, nullptr, nullptr ); new( proxy ) PyListBasketInputProxy( node, id, shape ); return proxy; } @@ -264,128 +265,128 @@ PyListBasketInputProxy * PyListBasketInputProxy::create( PyNode * node, INOUT_ID static void PyListBasketInputProxy_dealloc( PyListBasketInputProxy * self ) { CSP_BEGIN_METHOD; - self -> ~PyListBasketInputProxy(); - Py_TYPE( self ) -> tp_free( self ); + self->~PyListBasketInputProxy(); + Py_TYPE( self )->tp_free( self ); CSP_RETURN; } -//proxy by index +// proxy by index static PyObject * PyListBasketInputProxy_getproxy( PyListBasketInputProxy * proxy, PyObject * idx ) { - CSP_BEGIN_METHOD; - PyObject * rv = proxy -> proxy( fromPython( idx ) ); - Py_INCREF( rv ); - return rv; - CSP_RETURN_NONE; + CSP_BEGIN_METHOD; + PyObject * rv = proxy->proxy( fromPython( idx ) ); + Py_INCREF( rv ); + return rv; + CSP_RETURN_NONE; } -//value by index ( [n] ) +// value by index ( [n] ) static PyObject * PyListBasketInputProxy_getvalue( PyListBasketInputProxy * proxy, Py_ssize_t idx ) { - CSP_BEGIN_METHOD; - auto * in_proxy = proxy -> proxy( idx ); - if( !in_proxy -> valid() ) - CSP_THROW( RuntimeException, "list basket element " << idx << " is not valid" ); + CSP_BEGIN_METHOD; + auto * in_proxy = proxy->proxy( idx ); + if( !in_proxy->valid() ) + CSP_THROW( RuntimeException, "list basket element " << idx << " is not valid" ); - return in_proxy -> lastValue(); - CSP_RETURN_NONE; + return in_proxy->lastValue(); + CSP_RETURN_NONE; } static PyMethodDef PyListBasketInputProxy_methods[] = { - { "set_buffering_policy", (PyCFunction) PyBaseBasketInputProxy_set_buffering_policy, METH_VARARGS | METH_KEYWORDS, "set basket buffering policy" }, - { "make_active", (PyCFunction) PyBaseBasketInputProxy_make_active, METH_NOARGS, "make input active" }, - { "make_passive", (PyCFunction) PyBaseBasketInputProxy_make_passive, METH_NOARGS, "make input passive" }, - { "validvalues", (PyCFunction) PyBaseBasketInputProxy_validvalues, METH_NOARGS, "iterator of all valid values" }, - { "tickedvalues", (PyCFunction) PyBaseBasketInputProxy_tickedvalues, METH_NOARGS, "iterator of all ticked values" }, - { "validkeys", (PyCFunction) PyBaseBasketInputProxy_validkeys, METH_NOARGS, "iterator of all valid input keys" }, - { "validitems", (PyCFunction) PyBaseBasketInputProxy_validitems, METH_NOARGS, "iterator of key,value tuples of all valid inputs" }, - { "tickedkeys", (PyCFunction) PyBaseBasketInputProxy_tickedkeys, METH_NOARGS, "iterator of all ticked input keys" }, - { "tickeditems", (PyCFunction) PyBaseBasketInputProxy_tickeditems, METH_NOARGS, "iterator of key,value tuples of all ticked inputs" }, - { "_proxy", (PyCFunction) PyListBasketInputProxy_getproxy, METH_O, "Access input proxy" }, - { NULL } -}; - + { "set_buffering_policy", (PyCFunction)PyBaseBasketInputProxy_set_buffering_policy, METH_VARARGS | METH_KEYWORDS, + "set basket buffering policy" }, + { "make_active", (PyCFunction)PyBaseBasketInputProxy_make_active, METH_NOARGS, "make input active" }, + { "make_passive", (PyCFunction)PyBaseBasketInputProxy_make_passive, METH_NOARGS, "make input passive" }, + { "validvalues", (PyCFunction)PyBaseBasketInputProxy_validvalues, METH_NOARGS, "iterator of all valid values" }, + { "tickedvalues", (PyCFunction)PyBaseBasketInputProxy_tickedvalues, METH_NOARGS, "iterator of all ticked values" }, + { "validkeys", (PyCFunction)PyBaseBasketInputProxy_validkeys, METH_NOARGS, + "iterator of all valid input keys" }, + { "validitems", (PyCFunction)PyBaseBasketInputProxy_validitems, METH_NOARGS, + "iterator of key,value tuples of all valid inputs" }, + { "tickedkeys", (PyCFunction)PyBaseBasketInputProxy_tickedkeys, METH_NOARGS, + "iterator of all ticked input keys" }, + { "tickeditems", (PyCFunction)PyBaseBasketInputProxy_tickeditems, METH_NOARGS, + "iterator of key,value tuples of all ticked inputs" }, + { "_proxy", (PyCFunction)PyListBasketInputProxy_getproxy, METH_O, "Access input proxy" }, + { NULL } }; static PySequenceMethods PyListBasketInputProxy_SeqMethods = { - (lenfunc) PyBaseBasketInputProxy_len, /*sq_length */ - 0, /*sq_concat */ - 0, /*sq_repeat */ - (ssizeargfunc) PyListBasketInputProxy_getvalue, /*sq_item */ - 0, /*was_sq_slice */ - 0, /*sq_ass_item */ - 0, /*was_sq_ass_slice */ - 0 /*sq_contains */ + (lenfunc)PyBaseBasketInputProxy_len, /*sq_length */ + 0, /*sq_concat */ + 0, /*sq_repeat */ + (ssizeargfunc)PyListBasketInputProxy_getvalue, /*sq_item */ + 0, /*was_sq_slice */ + 0, /*sq_ass_item */ + 0, /*was_sq_ass_slice */ + 0 /*sq_contains */ }; static PyNumberMethods PyListBasketInputProxy_NumberMethods = { - 0, /* binaryfunc nb_add */ - 0, /* binaryfunc nb_subtract */ - 0, /* binaryfunc nb_multiply */ - 0, /* binaryfunc nb_remainder */ - 0, /* binaryfunc nb_divmod */ - 0, /* ternaryfunc nb_power */ - ( unaryfunc ) PyBaseBasketInputProxy_valid, /* unaryfunc nb_negative */ - ( unaryfunc ) PyBaseBasketInputProxy_ticked, /* unaryfunc nb_positive */ - 0, /* unaryfunc nb_absolute */ - 0, /* inquiry nb_nonzero */ - 0, /* unaryfunc nb_invert */ - 0, /* binaryfunc nb_lshift */ - 0, /* binaryfunc nb_rshift */ - 0, /* binaryfunc nb_and */ - 0, /* binaryfunc nb_xor */ - 0, /* binaryfunc nb_or */ - 0, /* unaryfunc nb_int */ - 0, /* void * reserved */ - 0, /* unaryfunc nb_float */ + 0, /* binaryfunc nb_add */ + 0, /* binaryfunc nb_subtract */ + 0, /* binaryfunc nb_multiply */ + 0, /* binaryfunc nb_remainder */ + 0, /* binaryfunc nb_divmod */ + 0, /* ternaryfunc nb_power */ + (unaryfunc)PyBaseBasketInputProxy_valid, /* unaryfunc nb_negative */ + (unaryfunc)PyBaseBasketInputProxy_ticked, /* unaryfunc nb_positive */ + 0, /* unaryfunc nb_absolute */ + 0, /* inquiry nb_nonzero */ + 0, /* unaryfunc nb_invert */ + 0, /* binaryfunc nb_lshift */ + 0, /* binaryfunc nb_rshift */ + 0, /* binaryfunc nb_and */ + 0, /* binaryfunc nb_xor */ + 0, /* binaryfunc nb_or */ + 0, /* unaryfunc nb_int */ + 0, /* void * reserved */ + 0, /* unaryfunc nb_float */ }; - PyTypeObject PyListBasketInputProxy::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyListBasketInputProxy", /* tp_name */ - sizeof(PyListBasketInputProxy), /* tp_basicsize */ - 0, /* tp_itemsize */ - ( destructor ) PyListBasketInputProxy_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - &PyListBasketInputProxy_NumberMethods,/* tp_as_number */ - &PyListBasketInputProxy_SeqMethods, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp list input proxy", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyListBasketInputProxy_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyListBasketInputProxy", /* tp_name */ + sizeof( PyListBasketInputProxy ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyListBasketInputProxy_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + &PyListBasketInputProxy_NumberMethods, /* tp_as_number */ + &PyListBasketInputProxy_SeqMethods, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp list input proxy", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyListBasketInputProxy_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ }; - -//DictBasket specific methods -PyDictBasketInputProxy::PyDictBasketInputProxy( PyNode * node, INOUT_ID_TYPE id, - PyObject * shape ) : PyBaseBasketInputProxy( node, id ), - m_shape( PyObjectPtr::incref( shape ) ) +// DictBasket specific methods +PyDictBasketInputProxy::PyDictBasketInputProxy( PyNode * node, INOUT_ID_TYPE id, PyObject * shape ) + : PyBaseBasketInputProxy( node, id ) + , m_shape( PyObjectPtr::incref( shape ) ) { m_proxyMapping = PyObjectPtr::own( PyDict_New() ); if( !m_proxyMapping.ptr() ) @@ -404,13 +405,14 @@ PyDictBasketInputProxy::PyDictBasketInputProxy( PyNode * node, INOUT_ID_TYPE id, PyDictBasketInputProxy * PyDictBasketInputProxy::create( PyNode * node, INOUT_ID_TYPE id, PyObject * shape ) { if( !PyList_Check( shape ) ) - CSP_THROW( TypeError, "Invalid shape for dict basket, expect list got: " << Py_TYPE( shape ) -> tp_name ); + CSP_THROW( TypeError, "Invalid shape for dict basket, expect list got: " << Py_TYPE( shape )->tp_name ); - if( ( size_t ) PyList_GET_SIZE( shape ) > InputId::maxBasketElements() ) - CSP_THROW( ValueError, "Dict basket size of " << PyList_GET_SIZE( shape ) << " exceeds basket size limit of " << - InputId::maxBasketElements() << " in node " << node -> name() ); + if( (size_t)PyList_GET_SIZE( shape ) > InputId::maxBasketElements() ) + CSP_THROW( ValueError, + "Dict basket size of " << PyList_GET_SIZE( shape ) << " exceeds basket size limit of " + << InputId::maxBasketElements() << " in node " << node->name() ); - PyDictBasketInputProxy * proxy = ( PyDictBasketInputProxy * ) PyType.tp_new( &PyType, nullptr, nullptr ); + PyDictBasketInputProxy * proxy = (PyDictBasketInputProxy *)PyType.tp_new( &PyType, nullptr, nullptr ); new( proxy ) PyDictBasketInputProxy( node, id, shape ); return proxy; } @@ -418,8 +420,8 @@ PyDictBasketInputProxy * PyDictBasketInputProxy::create( PyNode * node, INOUT_ID static void PyDictBasketInputProxy_dealloc( PyDictBasketInputProxy * self ) { CSP_BEGIN_METHOD; - self -> ~PyDictBasketInputProxy(); - Py_TYPE( self ) -> tp_free( self ); + self->~PyDictBasketInputProxy(); + Py_TYPE( self )->tp_free( self ); CSP_RETURN; } @@ -431,161 +433,160 @@ PyInputProxy * PyDictBasketInputProxy::proxyByKey( PyObject * key ) return in_proxy; } -//proxy by key +// proxy by key static PyObject * PyDictBasketInputProxy_getproxy( PyDictBasketInputProxy * proxy, PyObject * key ) { - CSP_BEGIN_METHOD; - PyObject * rv = proxy -> proxyByKey( key ); - Py_INCREF( rv ); - return rv; - CSP_RETURN_NONE; + CSP_BEGIN_METHOD; + PyObject * rv = proxy->proxyByKey( key ); + Py_INCREF( rv ); + return rv; + CSP_RETURN_NONE; } -//value by key ( [k] ) +// value by key ( [k] ) static PyObject * PyDictBasketInputProxy_getvalue( PyDictBasketInputProxy * proxy, PyObject * key ) { - CSP_BEGIN_METHOD; - auto * in_proxy = proxy -> proxyByKey( key ); - if( !in_proxy -> valid() ) - CSP_THROW( RuntimeException, "dict basket element " << PyObjectPtr::incref( key ) << " is not valid" ); + CSP_BEGIN_METHOD; + auto * in_proxy = proxy->proxyByKey( key ); + if( !in_proxy->valid() ) + CSP_THROW( RuntimeException, "dict basket element " << PyObjectPtr::incref( key ) << " is not valid" ); - return in_proxy -> lastValue(); - CSP_RETURN_NONE; + return in_proxy->lastValue(); + CSP_RETURN_NONE; } -bool PyDictBasketInputProxy::contains( PyObject * key ) -{ - return PyDict_Contains( m_proxyMapping.ptr(), key ); -} +bool PyDictBasketInputProxy::contains( PyObject * key ) { return PyDict_Contains( m_proxyMapping.ptr(), key ); } static bool PyDictBasketInputProxy_contains( PyDictBasketInputProxy * proxy, PyObject * key ) { - return proxy -> contains( key ); + return proxy->contains( key ); } static PyObject * PyDictBasketInputProxy_keys( PyDictBasketInputProxy * proxy ) { - PyObject * keys = proxy -> shape(); + PyObject * keys = proxy->shape(); Py_INCREF( keys ); return keys; } static PyMethodDef PyDictBasketInputProxy_methods[] = { - { "set_buffering_policy", (PyCFunction) PyBaseBasketInputProxy_set_buffering_policy, METH_VARARGS | METH_KEYWORDS, "set basket buffering policy" }, - { "make_active", (PyCFunction) PyBaseBasketInputProxy_make_active, METH_NOARGS, "make input active" }, - { "make_passive", (PyCFunction) PyBaseBasketInputProxy_make_passive, METH_NOARGS, "make input passive" }, - { "validvalues", (PyCFunction) PyBaseBasketInputProxy_validvalues, METH_NOARGS, "iterator of all valid values" }, - { "tickedvalues", (PyCFunction) PyBaseBasketInputProxy_tickedvalues, METH_NOARGS, "iterator of all ticked values" }, - { "validkeys", (PyCFunction) PyBaseBasketInputProxy_validkeys, METH_NOARGS, "iterator of all valid input keys" }, - { "validitems", (PyCFunction) PyBaseBasketInputProxy_validitems, METH_NOARGS, "iterator of key,value tuples of all valid inputs" }, - { "tickedkeys", (PyCFunction) PyBaseBasketInputProxy_tickedkeys, METH_NOARGS, "iterator of all ticked input keys" }, - { "tickeditems", (PyCFunction) PyBaseBasketInputProxy_tickeditems, METH_NOARGS, "iterator of key,value tuples of all ticked inputs" }, - { "keys", (PyCFunction) PyDictBasketInputProxy_keys, METH_NOARGS, "list of all keys on the basket" }, - { "_proxy", (PyCFunction) PyDictBasketInputProxy_getproxy, METH_O, "Access input proxy" }, - { NULL } -}; - + { "set_buffering_policy", (PyCFunction)PyBaseBasketInputProxy_set_buffering_policy, METH_VARARGS | METH_KEYWORDS, + "set basket buffering policy" }, + { "make_active", (PyCFunction)PyBaseBasketInputProxy_make_active, METH_NOARGS, "make input active" }, + { "make_passive", (PyCFunction)PyBaseBasketInputProxy_make_passive, METH_NOARGS, "make input passive" }, + { "validvalues", (PyCFunction)PyBaseBasketInputProxy_validvalues, METH_NOARGS, "iterator of all valid values" }, + { "tickedvalues", (PyCFunction)PyBaseBasketInputProxy_tickedvalues, METH_NOARGS, "iterator of all ticked values" }, + { "validkeys", (PyCFunction)PyBaseBasketInputProxy_validkeys, METH_NOARGS, + "iterator of all valid input keys" }, + { "validitems", (PyCFunction)PyBaseBasketInputProxy_validitems, METH_NOARGS, + "iterator of key,value tuples of all valid inputs" }, + { "tickedkeys", (PyCFunction)PyBaseBasketInputProxy_tickedkeys, METH_NOARGS, + "iterator of all ticked input keys" }, + { "tickeditems", (PyCFunction)PyBaseBasketInputProxy_tickeditems, METH_NOARGS, + "iterator of key,value tuples of all ticked inputs" }, + { "keys", (PyCFunction)PyDictBasketInputProxy_keys, METH_NOARGS, "list of all keys on the basket" }, + { "_proxy", (PyCFunction)PyDictBasketInputProxy_getproxy, METH_O, "Access input proxy" }, + { NULL } }; static PyMappingMethods PyDictBasketInputProxy_MappingMethods = { - (lenfunc) PyBaseBasketInputProxy_len, /*mp_length */ - (binaryfunc) PyDictBasketInputProxy_getvalue, /*mp_subscript */ - 0 /*mp_ass_subscript */ + (lenfunc)PyBaseBasketInputProxy_len, /*mp_length */ + (binaryfunc)PyDictBasketInputProxy_getvalue, /*mp_subscript */ + 0 /*mp_ass_subscript */ }; static PySequenceMethods PyDictBasketInputProxy_SeqMethods = { - 0, /*sq_length */ - 0, /*sq_concat */ - 0, /*sq_repeat */ - 0, /*sq_item */ - 0, /*was_sq_slice */ - 0, /*sq_ass_item */ - 0, /*was_sq_ass_slice */ - (objobjproc) PyDictBasketInputProxy_contains /*sq_contains */ + 0, /*sq_length */ + 0, /*sq_concat */ + 0, /*sq_repeat */ + 0, /*sq_item */ + 0, /*was_sq_slice */ + 0, /*sq_ass_item */ + 0, /*was_sq_ass_slice */ + (objobjproc)PyDictBasketInputProxy_contains /*sq_contains */ }; static PyNumberMethods PyDictBasketInputProxy_NumberMethods = { - 0, /* binaryfunc nb_add */ - 0, /* binaryfunc nb_subtract */ - 0, /* binaryfunc nb_multiply */ - 0, /* binaryfunc nb_remainder */ - 0, /* binaryfunc nb_divmod */ - 0, /* ternaryfunc nb_power */ - ( unaryfunc ) PyBaseBasketInputProxy_valid, /* unaryfunc nb_negative */ - ( unaryfunc ) PyBaseBasketInputProxy_ticked, /* unaryfunc nb_positive */ - 0, /* unaryfunc nb_absolute */ - 0, /* inquiry nb_nonzero */ - 0, /* unaryfunc nb_invert */ - 0, /* binaryfunc nb_lshift */ - 0, /* binaryfunc nb_rshift */ - 0, /* binaryfunc nb_and */ - 0, /* binaryfunc nb_xor */ - 0, /* binaryfunc nb_or */ - 0, /* unaryfunc nb_int */ - 0, /* void * reserved */ - 0, /* unaryfunc nb_float */ + 0, /* binaryfunc nb_add */ + 0, /* binaryfunc nb_subtract */ + 0, /* binaryfunc nb_multiply */ + 0, /* binaryfunc nb_remainder */ + 0, /* binaryfunc nb_divmod */ + 0, /* ternaryfunc nb_power */ + (unaryfunc)PyBaseBasketInputProxy_valid, /* unaryfunc nb_negative */ + (unaryfunc)PyBaseBasketInputProxy_ticked, /* unaryfunc nb_positive */ + 0, /* unaryfunc nb_absolute */ + 0, /* inquiry nb_nonzero */ + 0, /* unaryfunc nb_invert */ + 0, /* binaryfunc nb_lshift */ + 0, /* binaryfunc nb_rshift */ + 0, /* binaryfunc nb_and */ + 0, /* binaryfunc nb_xor */ + 0, /* binaryfunc nb_or */ + 0, /* unaryfunc nb_int */ + 0, /* void * reserved */ + 0, /* unaryfunc nb_float */ }; - PyTypeObject PyDictBasketInputProxy::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyDictBasketInputProxy", /* tp_name */ - sizeof(PyDictBasketInputProxy), /* tp_basicsize */ - 0, /* tp_itemsize */ - ( destructor ) PyDictBasketInputProxy_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - &PyDictBasketInputProxy_NumberMethods,/* tp_as_number */ - &PyDictBasketInputProxy_SeqMethods, /* tp_as_sequence */ - &PyDictBasketInputProxy_MappingMethods, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - "csp dict input proxy", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyDictBasketInputProxy_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyDictBasketInputProxy", /* tp_name */ + sizeof( PyDictBasketInputProxy ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyDictBasketInputProxy_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + &PyDictBasketInputProxy_NumberMethods, /* tp_as_number */ + &PyDictBasketInputProxy_SeqMethods, /* tp_as_sequence */ + &PyDictBasketInputProxy_MappingMethods, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + "csp dict input proxy", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyDictBasketInputProxy_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ }; -PyDynamicBasketInputProxy::PyDynamicBasketInputProxy( PyNode * node, INOUT_ID_TYPE id, PyObject * shape ) : PyDictBasketInputProxy( node, id, shape ) +PyDynamicBasketInputProxy::PyDynamicBasketInputProxy( PyNode * node, INOUT_ID_TYPE id, PyObject * shape ) + : PyDictBasketInputProxy( node, id, shape ) { - auto * dynBasket = const_cast( static_cast( basketInfo() ) ); - dynBasket -> setChangeCallback( [this]( const DialectGenericType & key, bool added, int64_t elemId, int64_t replaceId ) - { - handleShapeChange( key, added, elemId, replaceId ); - } ); + auto * dynBasket + = const_cast( static_cast( basketInfo() ) ); + dynBasket->setChangeCallback( [this]( const DialectGenericType & key, bool added, int64_t elemId, + int64_t replaceId ) { handleShapeChange( key, added, elemId, replaceId ); } ); m_shapeProxy = PyInputProxyPtr::own( PyInputProxy::create( m_node, InputId( m_id, -1 ) ) ); } PyDynamicBasketInputProxy * PyDynamicBasketInputProxy::create( PyNode * node, INOUT_ID_TYPE id ) { - PyObjectPtr shape = PyObjectPtr::own( PyList_New( 0 ) ); - PyDynamicBasketInputProxy * proxy = ( PyDynamicBasketInputProxy * ) PyType.tp_new( &PyType, nullptr, nullptr ); + PyObjectPtr shape = PyObjectPtr::own( PyList_New( 0 ) ); + PyDynamicBasketInputProxy * proxy = (PyDynamicBasketInputProxy *)PyType.tp_new( &PyType, nullptr, nullptr ); new( proxy ) PyDynamicBasketInputProxy( node, id, shape.get() ); return proxy; } -void PyDynamicBasketInputProxy::handleShapeChange( const DialectGenericType & key, bool added, int64_t elemId, int64_t replaceId ) +void PyDynamicBasketInputProxy::handleShapeChange( const DialectGenericType & key, bool added, int64_t elemId, + int64_t replaceId ) { auto & pyKey = reinterpret_cast( key ); @@ -596,7 +597,7 @@ void PyDynamicBasketInputProxy::handleShapeChange( const DialectGenericType & ke if( PyDict_SetItem( m_proxyMapping.ptr(), pyKey.get(), proxy.get() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - //current impl elements are always added at the end + // current impl elements are always added at the end CSP_ASSERT( elemId == PyList_GET_SIZE( m_shape.get() ) ); PyList_Append( m_shape.get(), pyKey.get() ); } @@ -605,32 +606,33 @@ void PyDynamicBasketInputProxy::handleShapeChange( const DialectGenericType & ke if( PyDict_DelItem( m_proxyMapping.ptr(), pyKey.get() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - //update shape, assumption are made on algorithm used to update ids ( asserted below ) + // update shape, assumption are made on algorithm used to update ids ( asserted below ) if( replaceId != -1 ) { - CSP_ASSERT( replaceId == ( PyList_GET_SIZE( m_shape.get() ) -1 ) ); - //free up the key being deleted + CSP_ASSERT( replaceId == ( PyList_GET_SIZE( m_shape.get() ) - 1 ) ); + // free up the key being deleted Py_DECREF( PyList_GET_ITEM( m_shape.get(), elemId ) ); - //get borrowed ref to key that will take up the slot + // get borrowed ref to key that will take up the slot PyObject * replacedKey = PyList_GET_ITEM( m_shape.get(), replaceId ); - - //set elemId slot = replaceId slot in our proxy dictionary - PyInputProxy * replacedproxy = static_cast( PyDict_GetItem( m_proxyMapping.get(), replacedKey ) ); + + // set elemId slot = replaceId slot in our proxy dictionary + PyInputProxy * replacedproxy + = static_cast( PyDict_GetItem( m_proxyMapping.get(), replacedKey ) ); CSP_ASSERT( replacedproxy != nullptr ); - replacedproxy -> setElemId( elemId ); + replacedproxy->setElemId( elemId ); - //set replaced key in new location in shape + // set replaced key in new location in shape PyList_SET_ITEM( m_shape.get(), elemId, replacedKey ); } else { - CSP_ASSERT( elemId == ( PyList_GET_SIZE( m_shape.get() ) -1 ) ); + CSP_ASSERT( elemId == ( PyList_GET_SIZE( m_shape.get() ) - 1 ) ); Py_DECREF( PyList_GET_ITEM( m_shape.get(), elemId ) ); } - //force resize down of last element. There is no C-API to do this for some reason.. but this is what the python C code - //does internally on pop ( essentially ) + // force resize down of last element. There is no C-API to do this for some reason.. but this is what the + // python C code does internally on pop ( essentially ) #if IS_PRE_PYTHON_3_11 Py_SIZE( m_shape.get() ) -= 1; #else @@ -642,77 +644,74 @@ void PyDynamicBasketInputProxy::handleShapeChange( const DialectGenericType & ke static void PyDynamicBasketInputProxy_dealloc( PyDynamicBasketInputProxy * self ) { CSP_BEGIN_METHOD; - self -> ~PyDynamicBasketInputProxy(); - Py_TYPE( self ) -> tp_free( self ); + self->~PyDynamicBasketInputProxy(); + Py_TYPE( self )->tp_free( self ); CSP_RETURN; } PyObject * PyDynamicBasketInputProxy::shape_getter( PyDynamicBasketInputProxy * self, void * ) { - return self -> m_shapeProxy -> lastValue(); + return self->m_shapeProxy->lastValue(); } PyObject * PyDynamicBasketInputProxy::shapeproxy_getter( PyDynamicBasketInputProxy * self, void * ) { - return self -> m_shapeProxy.incref().release(); + return self->m_shapeProxy.incref().release(); } -//ugh, python 3.6 PyGetSet strings are incorrectly declre char * instead of const char *... +// ugh, python 3.6 PyGetSet strings are incorrectly declre char * instead of const char *... static char s_shapename[] = "shape"; -static char s_shapedoc[] = "timeseries of the dynamic basket shape ts[csp.DynamicBasketEvents]"; +static char s_shapedoc[] = "timeseries of the dynamic basket shape ts[csp.DynamicBasketEvents]"; static char s_shapeproxyname[] = "_shapeproxy"; -static PyGetSetDef PyDynamicBasketInputProxy_getset[] = { - { s_shapename, ( getter ) PyDynamicBasketInputProxy::shape_getter, NULL, s_shapedoc, NULL }, - { s_shapeproxyname, ( getter ) PyDynamicBasketInputProxy::shapeproxy_getter, NULL, NULL, NULL }, - { NULL } -}; - +static PyGetSetDef PyDynamicBasketInputProxy_getset[] + = { { s_shapename, (getter)PyDynamicBasketInputProxy::shape_getter, NULL, s_shapedoc, NULL }, + { s_shapeproxyname, (getter)PyDynamicBasketInputProxy::shapeproxy_getter, NULL, NULL, NULL }, + { NULL } }; PyTypeObject PyDynamicBasketInputProxy::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyDynamicBasketInputProxy", /* tp_name */ - sizeof(PyDynamicBasketInputProxy), /* tp_basicsize */ - 0, /* tp_itemsize */ - ( destructor ) PyDynamicBasketInputProxy_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp dynamic basket input proxy", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - PyDynamicBasketInputProxy_getset, /* tp_getset */ - &PyDictBasketInputProxy::PyType, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyDynamicBasketInputProxy", /* tp_name */ + sizeof( PyDynamicBasketInputProxy ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyDynamicBasketInputProxy_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp dynamic basket input proxy", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + PyDynamicBasketInputProxy_getset, /* tp_getset */ + &PyDictBasketInputProxy::PyType, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ }; -REGISTER_TYPE_INIT( &PyListBasketInputProxy::PyType, "PyListBasketInputProxy" ); -REGISTER_TYPE_INIT( &PyDictBasketInputProxy::PyType, "PyDictBasketInputProxy" ); +REGISTER_TYPE_INIT( &PyListBasketInputProxy::PyType, "PyListBasketInputProxy" ); +REGISTER_TYPE_INIT( &PyDictBasketInputProxy::PyType, "PyDictBasketInputProxy" ); REGISTER_TYPE_INIT( &PyDynamicBasketInputProxy::PyType, "PyDynamicBasketInputProxy" ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyBasketOutputProxy.cpp b/cpp/csp/python/PyBasketOutputProxy.cpp index 474d9e49..941233ec 100644 --- a/cpp/csp/python/PyBasketOutputProxy.cpp +++ b/cpp/csp/python/PyBasketOutputProxy.cpp @@ -5,30 +5,36 @@ #include #include #include -#include #include #include +#include namespace csp::python { -PyBaseBasketOutputProxy::PyBaseBasketOutputProxy( Node * node, INOUT_ID_TYPE id ) : m_node( node ), - m_id( id ) +PyBaseBasketOutputProxy::PyBaseBasketOutputProxy( Node * node, INOUT_ID_TYPE id ) + : m_node( node ) + , m_id( id ) { } -PyListBasketOutputProxy::PyListBasketOutputProxy( PyObject *pyType, Node *node, INOUT_ID_TYPE id, size_t shape ) : PyBaseBasketOutputProxy( node, id ) +PyListBasketOutputProxy::PyListBasketOutputProxy( PyObject * pyType, Node * node, INOUT_ID_TYPE id, size_t shape ) + : PyBaseBasketOutputProxy( node, id ) { for( size_t elemId = 0; elemId < shape; ++elemId ) - m_proxies.emplace_back( PyOutputProxyPtr::own( PyOutputProxy::create( pyType, node, OutputId( id, elemId ) ) ) ); + m_proxies.emplace_back( + PyOutputProxyPtr::own( PyOutputProxy::create( pyType, node, OutputId( id, elemId ) ) ) ); } -PyListBasketOutputProxy * PyListBasketOutputProxy::create( PyObject *pyType, Node *node, INOUT_ID_TYPE id, size_t shape ) +PyListBasketOutputProxy * PyListBasketOutputProxy::create( PyObject * pyType, Node * node, INOUT_ID_TYPE id, + size_t shape ) { if( shape > OutputId::maxBasketElements() ) - CSP_THROW( ValueError, "List basket size of " << shape << " exceeds basket size limit of " << OutputId::maxBasketElements() << " in node " << node -> name() ); + CSP_THROW( ValueError, + "List basket size of " << shape << " exceeds basket size limit of " << OutputId::maxBasketElements() + << " in node " << node->name() ); - PyListBasketOutputProxy * proxy = ( PyListBasketOutputProxy * ) PyType.tp_new( &PyType, nullptr, nullptr ); + PyListBasketOutputProxy * proxy = (PyListBasketOutputProxy *)PyType.tp_new( &PyType, nullptr, nullptr ); new( proxy ) PyListBasketOutputProxy( pyType, node, id, shape ); return proxy; } @@ -36,15 +42,15 @@ PyListBasketOutputProxy * PyListBasketOutputProxy::create( PyObject *pyType, Nod static void PyListBasketOutputProxy_dealloc( PyListBasketOutputProxy * self ) { CSP_BEGIN_METHOD; - self -> ~PyListBasketOutputProxy(); - Py_TYPE( self ) -> tp_free( self ); + self->~PyListBasketOutputProxy(); + Py_TYPE( self )->tp_free( self ); CSP_RETURN; } static PyObject * PyListBasketOutputProxy_getproxy( PyListBasketOutputProxy * proxy, Py_ssize_t idx ) { CSP_BEGIN_METHOD; - PyObject * rv = proxy -> proxy( idx ); + PyObject * rv = proxy->proxy( idx ); Py_INCREF( rv ); return rv; CSP_RETURN_NONE; @@ -53,102 +59,102 @@ static PyObject * PyListBasketOutputProxy_getproxy( PyListBasketOutputProxy * pr static PyObject * PyListBasketOutputProxy_output( PyListBasketOutputProxy * proxy, PyObject * obj ) { CSP_BEGIN_METHOD; - if( !PyDict_Check(obj) ) - CSP_THROW( TypeError, "output called on list basket output proxy with non dict object: " << PyObjectPtr::incref( obj ) ); + if( !PyDict_Check( obj ) ) + CSP_THROW( TypeError, + "output called on list basket output proxy with non dict object: " << PyObjectPtr::incref( obj ) ); PyObject *key, *value; Py_ssize_t pos = 0; - while( PyDict_Next(obj, &pos, &key, &value) ) + while( PyDict_Next( obj, &pos, &key, &value ) ) { - if( !PyIndex_Check(key) ) + if( !PyIndex_Check( key ) ) CSP_THROW( TypeError, "output called on list basket with non-index key: " << PyObjectPtr::incref( key ) ); Py_ssize_t index = PyNumber_AsSsize_t( key, PyExc_IndexError ); if( index == -1 ) CSP_THROW( PythonPassthrough, "" ); - PyOutputProxy * outproxy = proxy -> proxy( index ); - outproxy -> outputTick( value ); + PyOutputProxy * outproxy = proxy->proxy( index ); + outproxy->outputTick( value ); } CSP_RETURN_NONE; } static PyNumberMethods PyListBasketOutputProxy_NumberMethods = { - (binaryfunc ) PyListBasketOutputProxy_output, /* binaryfunc nb_add */ - 0, /* binaryfunc nb_subtract */ - 0, /* binaryfunc nb_multiply */ - 0, /* binaryfunc nb_remainder */ - 0, /* binaryfunc nb_divmod */ - 0, /* ternaryfunc nb_power */ - 0, /* unaryfunc nb_negative */ - 0, /* unaryfunc nb_positive */ - 0, /* unaryfunc nb_absolute */ - 0, /* inquiry nb_nonzero */ - 0, /* unaryfunc nb_invert */ - 0, /* binaryfunc nb_lshift */ - 0, /* binaryfunc nb_rshift */ - 0, /* binaryfunc nb_and */ - 0, /* binaryfunc nb_xor */ - 0, /* binaryfunc nb_or */ - 0, /* unaryfunc nb_int */ - 0, /* void * reserved */ - 0 /* unaryfunc nb_float */ + (binaryfunc)PyListBasketOutputProxy_output, /* binaryfunc nb_add */ + 0, /* binaryfunc nb_subtract */ + 0, /* binaryfunc nb_multiply */ + 0, /* binaryfunc nb_remainder */ + 0, /* binaryfunc nb_divmod */ + 0, /* ternaryfunc nb_power */ + 0, /* unaryfunc nb_negative */ + 0, /* unaryfunc nb_positive */ + 0, /* unaryfunc nb_absolute */ + 0, /* inquiry nb_nonzero */ + 0, /* unaryfunc nb_invert */ + 0, /* binaryfunc nb_lshift */ + 0, /* binaryfunc nb_rshift */ + 0, /* binaryfunc nb_and */ + 0, /* binaryfunc nb_xor */ + 0, /* binaryfunc nb_or */ + 0, /* unaryfunc nb_int */ + 0, /* void * reserved */ + 0 /* unaryfunc nb_float */ }; static PySequenceMethods PyListBasketOutputProxy_SeqMethods = { - 0, /*sq_length */ - 0, /*sq_concat */ - 0, /*sq_repeat */ - (ssizeargfunc) PyListBasketOutputProxy_getproxy, /*sq_item */ - 0, /*sq_ass_item */ - 0, /*sq_contains */ - 0, /*sq_inplace_concat */ - 0 /*sq_inplace_repeat */ + 0, /*sq_length */ + 0, /*sq_concat */ + 0, /*sq_repeat */ + (ssizeargfunc)PyListBasketOutputProxy_getproxy, /*sq_item */ + 0, /*sq_ass_item */ + 0, /*sq_contains */ + 0, /*sq_inplace_concat */ + 0 /*sq_inplace_repeat */ }; PyTypeObject PyListBasketOutputProxy::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyListBasketOutputProxy", /* tp_name */ - sizeof(PyListBasketOutputProxy), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) PyListBasketOutputProxy_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - &PyListBasketOutputProxy_NumberMethods, /* tp_as_number */ - &PyListBasketOutputProxy_SeqMethods, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp list output proxy", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyListBasketOutputProxy", /* tp_name */ + sizeof( PyListBasketOutputProxy ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyListBasketOutputProxy_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + &PyListBasketOutputProxy_NumberMethods, /* tp_as_number */ + &PyListBasketOutputProxy_SeqMethods, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp list output proxy", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ }; -PyDictBasketOutputProxy::PyDictBasketOutputProxy( PyObject *pyType, Node *node, INOUT_ID_TYPE id, - PyObject * shape ) : PyBaseBasketOutputProxy( node, id ) +PyDictBasketOutputProxy::PyDictBasketOutputProxy( PyObject * pyType, Node * node, INOUT_ID_TYPE id, PyObject * shape ) + : PyBaseBasketOutputProxy( node, id ) { m_proxyMapping = PyObjectPtr::own( PyDict_New() ); if( !m_proxyMapping.ptr() ) @@ -160,20 +166,21 @@ PyDictBasketOutputProxy::PyDictBasketOutputProxy( PyObject *pyType, Node *node, auto proxy = PyOutputProxyPtr::own( PyOutputProxy::create( pyType, node, OutputId( id, elemId ) ) ); if( PyDict_SetItem( m_proxyMapping.ptr(), PyList_GET_ITEM( shape, elemId ), proxy.ptr() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - } } -PyDictBasketOutputProxy * PyDictBasketOutputProxy::create( PyObject *pyType, Node *node, INOUT_ID_TYPE id, PyObject * shape ) +PyDictBasketOutputProxy * PyDictBasketOutputProxy::create( PyObject * pyType, Node * node, INOUT_ID_TYPE id, + PyObject * shape ) { if( !PyList_Check( shape ) ) - CSP_THROW( TypeError, "Invalid shape for dict basket, expect list got: " << Py_TYPE( shape ) -> tp_name ); + CSP_THROW( TypeError, "Invalid shape for dict basket, expect list got: " << Py_TYPE( shape )->tp_name ); - if( ( size_t ) PyList_GET_SIZE( shape ) > OutputId::maxBasketElements() ) - CSP_THROW( ValueError, "Dict basket size of " << PyList_GET_SIZE( shape ) << " exceeds basket size limit of " << - OutputId::maxBasketElements() << " in node " << node -> name() ); + if( (size_t)PyList_GET_SIZE( shape ) > OutputId::maxBasketElements() ) + CSP_THROW( ValueError, + "Dict basket size of " << PyList_GET_SIZE( shape ) << " exceeds basket size limit of " + << OutputId::maxBasketElements() << " in node " << node->name() ); - PyDictBasketOutputProxy * proxy = ( PyDictBasketOutputProxy * ) PyType.tp_new( &PyType, nullptr, nullptr ); + PyDictBasketOutputProxy * proxy = (PyDictBasketOutputProxy *)PyType.tp_new( &PyType, nullptr, nullptr ); new( proxy ) PyDictBasketOutputProxy( pyType, node, id, shape ); return proxy; } @@ -181,8 +188,8 @@ PyDictBasketOutputProxy * PyDictBasketOutputProxy::create( PyObject *pyType, Nod static void PyDictBasketOutputProxy_dealloc( PyDictBasketOutputProxy * self ) { CSP_BEGIN_METHOD; - self -> ~PyDictBasketOutputProxy(); - Py_TYPE( self ) -> tp_free( self ); + self->~PyDictBasketOutputProxy(); + Py_TYPE( self )->tp_free( self ); CSP_RETURN; } @@ -197,7 +204,7 @@ PyOutputProxy * PyDictBasketOutputProxy::proxyByKey( PyObject * key ) static PyObject * PyDictBasketOutputProxy_getproxy( PyDictBasketOutputProxy * proxy, PyObject * key ) { CSP_BEGIN_METHOD; - PyObject * rv = proxy -> proxyByKey( key ); + PyObject * rv = proxy->proxyByKey( key ); Py_INCREF( rv ); return rv; CSP_RETURN_NONE; @@ -206,98 +213,99 @@ static PyObject * PyDictBasketOutputProxy_getproxy( PyDictBasketOutputProxy * pr static PyObject * PyDictBasketOutputProxy_output( PyDictBasketOutputProxy * proxy, PyObject * obj ) { CSP_BEGIN_METHOD; - if( !PyDict_Check(obj) ) - CSP_THROW( TypeError, "output called on dict basket output proxy with non dict object: " << PyObjectPtr::incref( obj ) ); + if( !PyDict_Check( obj ) ) + CSP_THROW( TypeError, + "output called on dict basket output proxy with non dict object: " << PyObjectPtr::incref( obj ) ); PyObject *key, *value; Py_ssize_t pos = 0; - while( PyDict_Next(obj, &pos, &key, &value) ) + while( PyDict_Next( obj, &pos, &key, &value ) ) { - PyOutputProxy * outproxy = proxy -> proxyByKey( key ); - outproxy -> outputTick( value ); + PyOutputProxy * outproxy = proxy->proxyByKey( key ); + outproxy->outputTick( value ); } CSP_RETURN_NONE; } static PyNumberMethods PyDictBasketOutputProxy_NumberMethods = { - (binaryfunc ) PyDictBasketOutputProxy_output, /* binaryfunc nb_add */ - 0, /* binaryfunc nb_subtract */ - 0, /* binaryfunc nb_multiply */ - 0, /* binaryfunc nb_remainder */ - 0, /* binaryfunc nb_divmod */ - 0, /* ternaryfunc nb_power */ - 0, /* unaryfunc nb_negative */ - 0, /* unaryfunc nb_positive */ - 0, /* unaryfunc nb_absolute */ - 0, /* inquiry nb_nonzero */ - 0, /* unaryfunc nb_invert */ - 0, /* binaryfunc nb_lshift */ - 0, /* binaryfunc nb_rshift */ - 0, /* binaryfunc nb_and */ - 0, /* binaryfunc nb_xor */ - 0, /* binaryfunc nb_or */ - 0, /* unaryfunc nb_int */ - 0, /* void * reserved */ - 0 /* unaryfunc nb_float */ + (binaryfunc)PyDictBasketOutputProxy_output, /* binaryfunc nb_add */ + 0, /* binaryfunc nb_subtract */ + 0, /* binaryfunc nb_multiply */ + 0, /* binaryfunc nb_remainder */ + 0, /* binaryfunc nb_divmod */ + 0, /* ternaryfunc nb_power */ + 0, /* unaryfunc nb_negative */ + 0, /* unaryfunc nb_positive */ + 0, /* unaryfunc nb_absolute */ + 0, /* inquiry nb_nonzero */ + 0, /* unaryfunc nb_invert */ + 0, /* binaryfunc nb_lshift */ + 0, /* binaryfunc nb_rshift */ + 0, /* binaryfunc nb_and */ + 0, /* binaryfunc nb_xor */ + 0, /* binaryfunc nb_or */ + 0, /* unaryfunc nb_int */ + 0, /* void * reserved */ + 0 /* unaryfunc nb_float */ }; static PyMappingMethods PyDictBasketOutputProxy_MappingMethods = { - 0, /*mp_length */ - (binaryfunc) PyDictBasketOutputProxy_getproxy, /*mp_subscript */ - 0 /*mp_ass_subscript */ + 0, /*mp_length */ + (binaryfunc)PyDictBasketOutputProxy_getproxy, /*mp_subscript */ + 0 /*mp_ass_subscript */ }; PyTypeObject PyDictBasketOutputProxy::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyDictBasketOutputProxy", /* tp_name */ - sizeof(PyDictBasketOutputProxy), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) PyDictBasketOutputProxy_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - &PyDictBasketOutputProxy_NumberMethods, /* tp_as_number */ - 0, /* tp_as_sequence */ - &PyDictBasketOutputProxy_MappingMethods, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp dict output proxy", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyDictBasketOutputProxy", /* tp_name */ + sizeof( PyDictBasketOutputProxy ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyDictBasketOutputProxy_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + &PyDictBasketOutputProxy_NumberMethods, /* tp_as_number */ + 0, /* tp_as_sequence */ + &PyDictBasketOutputProxy_MappingMethods, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp dict output proxy", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ }; -PyDynamicBasketOutputProxy::PyDynamicBasketOutputProxy( PyObject *pyType, Node * node, - INOUT_ID_TYPE id, PyObject * shape ) : PyDictBasketOutputProxy( pyType, node, id, shape ) +PyDynamicBasketOutputProxy::PyDynamicBasketOutputProxy( PyObject * pyType, Node * node, INOUT_ID_TYPE id, + PyObject * shape ) + : PyDictBasketOutputProxy( pyType, node, id, shape ) { m_elemType = PyObjectPtr::incref( pyType ); } PyDynamicBasketOutputProxy * PyDynamicBasketOutputProxy::create( PyObject * pyType, Node * node, INOUT_ID_TYPE id ) { - PyObjectPtr shape = PyObjectPtr::own( PyList_New( 0 ) ); - PyDynamicBasketOutputProxy * proxy = ( PyDynamicBasketOutputProxy * ) PyType.tp_new( &PyType, nullptr, nullptr ); + PyObjectPtr shape = PyObjectPtr::own( PyList_New( 0 ) ); + PyDynamicBasketOutputProxy * proxy = (PyDynamicBasketOutputProxy *)PyType.tp_new( &PyType, nullptr, nullptr ); new( proxy ) PyDynamicBasketOutputProxy( pyType, node, id, shape.get() ); return proxy; } @@ -305,8 +313,8 @@ PyDynamicBasketOutputProxy * PyDynamicBasketOutputProxy::create( PyObject * pyTy static void PyDynamicBasketOutputProxy_dealloc( PyDynamicBasketOutputProxy * self ) { CSP_BEGIN_METHOD; - self -> ~PyDynamicBasketOutputProxy(); - Py_TYPE( self ) -> tp_free( self ); + self->~PyDynamicBasketOutputProxy(); + Py_TYPE( self )->tp_free( self ); CSP_RETURN; } @@ -315,18 +323,19 @@ PyOutputProxy * PyDynamicBasketOutputProxy::getOrCreateProxy( PyObject * key ) PyOutputProxy * outproxy = static_cast( PyDict_GetItem( m_proxyMapping.ptr(), key ) ); if( outproxy == nullptr ) { - //create the new entry - auto * dynBasket = static_cast( m_node -> outputBasket( m_id ) ); - auto elemId = dynBasket -> addDynamicKey( fromPython( key ) ); - auto proxy = PyOutputProxyPtr::own( PyOutputProxy::create( m_elemType.get(), m_node, OutputId( m_id, elemId ) ) ); + // create the new entry + auto * dynBasket = static_cast( m_node->outputBasket( m_id ) ); + auto elemId = dynBasket->addDynamicKey( fromPython( key ) ); + auto proxy + = PyOutputProxyPtr::own( PyOutputProxy::create( m_elemType.get(), m_node, OutputId( m_id, elemId ) ) ); if( PyDict_SetItem( m_proxyMapping.ptr(), key, proxy.ptr() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); outproxy = proxy.ptr(); - if( ( size_t ) elemId >= m_keyMapping.size() ) + if( (size_t)elemId >= m_keyMapping.size() ) m_keyMapping.resize( elemId + 1 ); - m_keyMapping[ elemId ] = PyObjectPtr::incref( key ); + m_keyMapping[elemId] = PyObjectPtr::incref( key ); } return static_cast( outproxy ); } @@ -335,123 +344,126 @@ void PyDynamicBasketOutputProxy::removeProxy( PyObject * key ) { PyOutputProxy * outproxy = static_cast( PyDict_GetItem( m_proxyMapping.ptr(), key ) ); if( outproxy == nullptr ) - CSP_THROW( KeyError, "attempting to remove unknown key " << PyObjectPtr::incref( key ) << " from dynamic basket" ); + CSP_THROW( KeyError, + "attempting to remove unknown key " << PyObjectPtr::incref( key ) << " from dynamic basket" ); - //Disallow removing timeseries that ticked in the same cycle - if( outproxy -> ts() -> lastCycleCount() == m_node -> cycleCount() ) - CSP_THROW( RuntimeException, "Attempted to delete dynamic basket key '" << PyObjectPtr::incref( key ) << "' which was already ticked this cycle" ); + // Disallow removing timeseries that ticked in the same cycle + if( outproxy->ts()->lastCycleCount() == m_node->cycleCount() ) + CSP_THROW( RuntimeException, + "Attempted to delete dynamic basket key '" << PyObjectPtr::incref( key ) + << "' which was already ticked this cycle" ); - auto elemId = outproxy -> outputId().elemId; + auto elemId = outproxy->outputId().elemId; if( PyDict_DelItem( m_proxyMapping.ptr(), key ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - auto * dynBasket = static_cast( m_node -> outputBasket( m_id ) ); - auto replaceId = dynBasket -> removeDynamicKey( fromPython( key ), elemId ); + auto * dynBasket = static_cast( m_node->outputBasket( m_id ) ); + auto replaceId = dynBasket->removeDynamicKey( fromPython( key ), elemId ); if( replaceId != -1 ) { - //need to find which key moved in the internal basket - auto & replaceKey = m_keyMapping[ replaceId ]; - PyOutputProxy * replacedproxy = static_cast( PyDict_GetItem( m_proxyMapping.ptr(), replaceKey.get() ) ); + // need to find which key moved in the internal basket + auto & replaceKey = m_keyMapping[replaceId]; + PyOutputProxy * replacedproxy + = static_cast( PyDict_GetItem( m_proxyMapping.ptr(), replaceKey.get() ) ); CSP_ASSERT( replacedproxy != nullptr ); - replacedproxy -> setElemId( elemId ); + replacedproxy->setElemId( elemId ); - m_keyMapping[ elemId ] = replaceKey; - m_keyMapping[ replaceId ].reset(); + m_keyMapping[elemId] = replaceKey; + m_keyMapping[replaceId].reset(); } else - m_keyMapping[ elemId ].reset(); + m_keyMapping[elemId].reset(); } static PyObject * PyDynamicBasketOutputProxy_output( PyDynamicBasketOutputProxy * proxy, PyObject * obj ) { CSP_BEGIN_METHOD; - if( !PyDict_Check(obj) ) - CSP_THROW( TypeError, "output called on dict basket output proxy with non dict object: " << PyObjectPtr::incref( obj ) ); + if( !PyDict_Check( obj ) ) + CSP_THROW( TypeError, + "output called on dict basket output proxy with non dict object: " << PyObjectPtr::incref( obj ) ); PyObject *key, *value; Py_ssize_t pos = 0; - while( PyDict_Next(obj, &pos, &key, &value) ) + while( PyDict_Next( obj, &pos, &key, &value ) ) { if( value == constants::REMOVE_DYNAMIC_KEY() ) - proxy -> removeProxy( key ); + proxy->removeProxy( key ); else { - auto * outproxy = proxy -> getOrCreateProxy( key ); - outproxy -> outputTick( value ); + auto * outproxy = proxy->getOrCreateProxy( key ); + outproxy->outputTick( value ); } } CSP_RETURN_NONE; } static PyNumberMethods PyDynamicBasketOutputProxy_NumberMethods = { - (binaryfunc ) PyDynamicBasketOutputProxy_output, /* binaryfunc nb_add */ - 0, /* binaryfunc nb_subtract */ - 0, /* binaryfunc nb_multiply */ - 0, /* binaryfunc nb_remainder */ - 0, /* binaryfunc nb_divmod */ - 0, /* ternaryfunc nb_power */ - 0, /* unaryfunc nb_negative */ - 0, /* unaryfunc nb_positive */ - 0, /* unaryfunc nb_absolute */ - 0, /* inquiry nb_nonzero */ - 0, /* unaryfunc nb_invert */ - 0, /* binaryfunc nb_lshift */ - 0, /* binaryfunc nb_rshift */ - 0, /* binaryfunc nb_and */ - 0, /* binaryfunc nb_xor */ - 0, /* binaryfunc nb_or */ - 0, /* unaryfunc nb_int */ - 0, /* void * reserved */ - 0 /* unaryfunc nb_float */ + (binaryfunc)PyDynamicBasketOutputProxy_output, /* binaryfunc nb_add */ + 0, /* binaryfunc nb_subtract */ + 0, /* binaryfunc nb_multiply */ + 0, /* binaryfunc nb_remainder */ + 0, /* binaryfunc nb_divmod */ + 0, /* ternaryfunc nb_power */ + 0, /* unaryfunc nb_negative */ + 0, /* unaryfunc nb_positive */ + 0, /* unaryfunc nb_absolute */ + 0, /* inquiry nb_nonzero */ + 0, /* unaryfunc nb_invert */ + 0, /* binaryfunc nb_lshift */ + 0, /* binaryfunc nb_rshift */ + 0, /* binaryfunc nb_and */ + 0, /* binaryfunc nb_xor */ + 0, /* binaryfunc nb_or */ + 0, /* unaryfunc nb_int */ + 0, /* void * reserved */ + 0 /* unaryfunc nb_float */ }; PyTypeObject PyDynamicBasketOutputProxy::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyDynamicBasketOutputProxy", /* tp_name */ - sizeof(PyDynamicBasketOutputProxy), /* tp_basicsize */ - 0, /* tp_itemsize */ - ( destructor ) PyDynamicBasketOutputProxy_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - &PyDynamicBasketOutputProxy_NumberMethods, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp dynamic basket output proxy", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyDictBasketOutputProxy::PyType, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyDynamicBasketOutputProxy", /* tp_name */ + sizeof( PyDynamicBasketOutputProxy ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyDynamicBasketOutputProxy_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + &PyDynamicBasketOutputProxy_NumberMethods, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp dynamic basket output proxy", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyDictBasketOutputProxy::PyType, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ }; - -REGISTER_TYPE_INIT( &PyListBasketOutputProxy::PyType, "PyListBasketOutputProxy" ); -REGISTER_TYPE_INIT( &PyDictBasketOutputProxy::PyType, "PyDictBasketOutputProxy" ); +REGISTER_TYPE_INIT( &PyListBasketOutputProxy::PyType, "PyListBasketOutputProxy" ); +REGISTER_TYPE_INIT( &PyDictBasketOutputProxy::PyType, "PyDictBasketOutputProxy" ); REGISTER_TYPE_INIT( &PyDynamicBasketOutputProxy::PyType, "PyDynamicBasketOutputProxy" ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyConstAdapter.cpp b/cpp/csp/python/PyConstAdapter.cpp index a37fce34..d871d155 100644 --- a/cpp/csp/python/PyConstAdapter.cpp +++ b/cpp/csp/python/PyConstAdapter.cpp @@ -1,40 +1,35 @@ #include #include -#include #include +#include #include namespace csp::python { -static InputAdapter * const_creator( csp::AdapterManager * manager, PyEngine * engine, - PyObject * pyType, PushMode pushMode, PyObject * args ) +static InputAdapter * const_creator( csp::AdapterManager * manager, PyEngine * engine, PyObject * pyType, + PushMode pushMode, PyObject * args ) { - PyObject * pyValue = nullptr; - PyObject * pyDelay = nullptr; + PyObject * pyValue = nullptr; + PyObject * pyDelay = nullptr; - if( !PyArg_ParseTuple( args, "OO", - &pyValue, - &pyDelay ) ) + if( !PyArg_ParseTuple( args, "OO", &pyValue, &pyDelay ) ) CSP_THROW( PythonPassthrough, "" ); auto delay = fromPython( pyDelay ); auto & cspType = pyTypeAsCspType( pyType ); - InputAdapter *adapter = switchCspType( cspType, - [ engine, &cspType, pyValue, delay ]( auto tag ) -> InputAdapter * - { - using T = typename decltype(tag)::type; - return engine -> engine() -> createOwnedObject>( cspType, - fromPython( - pyValue, - *cspType ), - delay ); - } ); + InputAdapter * adapter = switchCspType( cspType, + [engine, &cspType, pyValue, delay]( auto tag ) -> InputAdapter * + { + using T = typename decltype( tag )::type; + return engine->engine()->createOwnedObject>( + cspType, fromPython( pyValue, *cspType ), delay ); + } ); return adapter; } REGISTER_INPUT_ADAPTER( _const, const_creator ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyConstants.cpp b/cpp/csp/python/PyConstants.cpp index a92417b2..19a7c0d7 100644 --- a/cpp/csp/python/PyConstants.cpp +++ b/cpp/csp/python/PyConstants.cpp @@ -1,8 +1,8 @@ -#include -#include #include #include #include +#include +#include static PyObject * s_UNSET; static PyObject * s_REMOVE_DYNAMIC_KEY; @@ -10,40 +10,40 @@ static PyObject * s_EDGE_TYPE; namespace csp::python::constants { - PyObject * UNSET() +PyObject * UNSET() +{ + if( unlikely( s_UNSET == nullptr ) ) { - if( unlikely( s_UNSET == nullptr ) ) - { - PyObject * wiring = PyImport_AddModule("csp.impl.constants"); - PyObject * dict = PyModule_GetDict(wiring); - s_UNSET = PyDict_GetItemString(dict, "UNSET"); - Py_INCREF( s_UNSET ); - } - return s_UNSET; + PyObject * wiring = PyImport_AddModule( "csp.impl.constants" ); + PyObject * dict = PyModule_GetDict( wiring ); + s_UNSET = PyDict_GetItemString( dict, "UNSET" ); + Py_INCREF( s_UNSET ); } + return s_UNSET; +} - PyObject * REMOVE_DYNAMIC_KEY() +PyObject * REMOVE_DYNAMIC_KEY() +{ + if( unlikely( s_REMOVE_DYNAMIC_KEY == nullptr ) ) { - if( unlikely( s_REMOVE_DYNAMIC_KEY == nullptr ) ) - { - PyObject * wiring = PyImport_AddModule("csp.impl.constants"); - PyObject * dict = PyModule_GetDict(wiring); - s_REMOVE_DYNAMIC_KEY = PyDict_GetItemString(dict, "REMOVE_DYNAMIC_KEY"); - Py_INCREF( s_REMOVE_DYNAMIC_KEY ); - } - return s_REMOVE_DYNAMIC_KEY; + PyObject * wiring = PyImport_AddModule( "csp.impl.constants" ); + PyObject * dict = PyModule_GetDict( wiring ); + s_REMOVE_DYNAMIC_KEY = PyDict_GetItemString( dict, "REMOVE_DYNAMIC_KEY" ); + Py_INCREF( s_REMOVE_DYNAMIC_KEY ); } + return s_REMOVE_DYNAMIC_KEY; +} - PyObject * EDGE_TYPE() +PyObject * EDGE_TYPE() +{ + if( unlikely( s_EDGE_TYPE == nullptr ) ) { - if( unlikely( s_EDGE_TYPE == nullptr ) ) - { - PyObject * mod = PyImport_AddModule("csp.impl.wiring.edge"); - PyObject * dict = PyModule_GetDict( mod ); - s_EDGE_TYPE = PyDict_GetItemString( dict, "Edge" ); - Py_INCREF( s_EDGE_TYPE ); - } - - return s_EDGE_TYPE; + PyObject * mod = PyImport_AddModule( "csp.impl.wiring.edge" ); + PyObject * dict = PyModule_GetDict( mod ); + s_EDGE_TYPE = PyDict_GetItemString( dict, "Edge" ); + Py_INCREF( s_EDGE_TYPE ); } + + return s_EDGE_TYPE; } +} // namespace csp::python::constants diff --git a/cpp/csp/python/PyCppNode.cpp b/cpp/csp/python/PyCppNode.cpp index 85d5a426..bcff048d 100644 --- a/cpp/csp/python/PyCppNode.cpp +++ b/cpp/csp/python/PyCppNode.cpp @@ -7,27 +7,29 @@ namespace csp::python { -//not pytype is in/out value +// not pytype is in/out value CppNode::Shape extractShape( const char * nodeName, PyObject *& pytype ) { CppNode::Shape shape; if( PyTuple_Check( pytype ) ) { - PyObject * pyshape = (PyObject * ) PyTuple_GET_ITEM( pytype, 0 ); - pytype = PyTuple_GET_ITEM( pytype, 1 ); + PyObject * pyshape = (PyObject *)PyTuple_GET_ITEM( pytype, 0 ); + pytype = PyTuple_GET_ITEM( pytype, 1 ); if( PyLong_Check( pyshape ) ) { auto longShape = fromPython( pyshape ); - if( longShape > InputId::maxBasketElements() ) - CSP_THROW( ValueError, "basket size exceeds limit of " << InputId::maxBasketElements() << " on node \"" << nodeName << "\""); + if( longShape > InputId::maxBasketElements() ) + CSP_THROW( ValueError, + "basket size exceeds limit of " << InputId::maxBasketElements() << " on node \"" << nodeName + << "\"" ); shape = longShape; } else { if( !PyList_Check( pyshape ) ) - CSP_THROW( TypeError, "Expected basket shape as int or list, got " << Py_TYPE( pyshape ) -> tp_name ); + CSP_THROW( TypeError, "Expected basket shape as int or list, got " << Py_TYPE( pyshape )->tp_name ); std::vector keys; for( int i = 0; i < PyList_GET_SIZE( pyshape ); ++i ) { @@ -49,12 +51,8 @@ PyObject * pycppnode_create( PyObject * args, CppNode::Creator creatorFn ) PyObject * pyinputs; PyObject * pyoutputs; PyObject * pyscalars; - if( !PyArg_ParseTuple( args, "sO!O!O!O!", - &nodeName, - &csp::python::PyEngine::PyType, &pyengine, - &PyTuple_Type, &pyinputs, - &PyTuple_Type, &pyoutputs, - &PyDict_Type, &pyscalars ) ) + if( !PyArg_ParseTuple( args, "sO!O!O!O!", &nodeName, &csp::python::PyEngine::PyType, &pyengine, &PyTuple_Type, + &pyinputs, &PyTuple_Type, &pyoutputs, &PyDict_Type, &pyscalars ) ) CSP_THROW( PythonPassthrough, "" ); csp::CppNode::NodeDef nodedef; @@ -70,7 +68,7 @@ PyObject * pycppnode_create( PyObject * args, CppNode::Creator creatorFn ) CppNode::Shape shape = extractShape( nodeName, pytype ); - nodedef.inputs[ name ] = { INOUT_ID_TYPE( index ), pyTypeAsCspType( pytype ), ( bool ) isAlarm, shape }; + nodedef.inputs[name] = { INOUT_ID_TYPE( index ), pyTypeAsCspType( pytype ), (bool)isAlarm, shape }; } for( int i = 0; i < PyTuple_GET_SIZE( pyoutputs ); ++i ) @@ -78,13 +76,13 @@ PyObject * pycppnode_create( PyObject * args, CppNode::Creator creatorFn ) if( !PyArg_ParseTuple( PyTuple_GET_ITEM( pyoutputs, i ), "sOi", &name, &pytype, &index ) ) CSP_THROW( PythonPassthrough, "" ); - CppNode::Shape shape = extractShape( nodeName, pytype ); - nodedef.outputs[ name ] = { INOUT_ID_TYPE( index ), pyTypeAsCspType( pytype ), false, shape }; + CppNode::Shape shape = extractShape( nodeName, pytype ); + nodedef.outputs[name] = { INOUT_ID_TYPE( index ), pyTypeAsCspType( pytype ), false, shape }; } nodedef.scalars = fromPython( pyscalars ); - auto node = creatorFn( pyengine -> engine(), nodedef ); + auto node = creatorFn( pyengine->engine(), nodedef ); return PyNodeWrapper::create( node ); } -} +} // namespace csp::python diff --git a/cpp/csp/python/PyCspEnum.cpp b/cpp/csp/python/PyCspEnum.cpp index cf7c8269..bd0529e3 100644 --- a/cpp/csp/python/PyCspEnum.cpp +++ b/cpp/csp/python/PyCspEnum.cpp @@ -8,9 +8,9 @@ namespace csp::python { DialectCspEnumMeta::DialectCspEnumMeta( PyTypeObjectPtr pyType, const std::string & name, - const CspEnumMeta::ValueDef & def ) : - CspEnumMeta( name, def ), - m_pyType( pyType ) + const CspEnumMeta::ValueDef & def ) + : CspEnumMeta( name, def ) + , m_pyType( pyType ) { } @@ -18,28 +18,26 @@ DialectCspEnumMeta::DialectCspEnumMeta( PyTypeObjectPtr pyType, const std::strin MetaClass Madness NOTES!!! -- see PyStruct.cpp for note, same idea */ -static PyObject * PyCspEnumMeta_new( PyTypeObject *subtype, PyObject *args, PyObject *kwds ) +static PyObject * PyCspEnumMeta_new( PyTypeObject * subtype, PyObject * args, PyObject * kwds ) { CSP_BEGIN_METHOD; PyObject * pyname; PyObject * bases; PyObject * dict; - if( !PyArg_ParseTuple( args, "UO!O!", - &pyname, - &PyTuple_Type, &bases, - &PyDict_Type, &dict ) ) + if( !PyArg_ParseTuple( args, "UO!O!", &pyname, &PyTuple_Type, &bases, &PyDict_Type, &dict ) ) CSP_THROW( PythonPassthrough, "" ); - //subtype is python defined CspEnumMeta class - PyCspEnumMeta * pymeta = ( PyCspEnumMeta * ) PyType_Type.tp_new( subtype, args, kwds ); + // subtype is python defined CspEnumMeta class + PyCspEnumMeta * pymeta = (PyCspEnumMeta *)PyType_Type.tp_new( subtype, args, kwds ); - //Note that we call ctor without parents so as not to 0-init the base POD PyTypeObject class after its been initialized - new ( pymeta ) PyCspEnumMeta; + // Note that we call ctor without parents so as not to 0-init the base POD PyTypeObject class after its been + // initialized + new( pymeta ) PyCspEnumMeta; - //this would be the CspEnum class on python side, it doesnt create any metadata for itself - if( pymeta -> ht_type.tp_base == &PyCspEnum::PyType ) - return ( PyObject * ) pymeta; + // this would be the CspEnum class on python side, it doesnt create any metadata for itself + if( pymeta->ht_type.tp_base == &PyCspEnum::PyType ) + return (PyObject *)pymeta; std::string name = PyUnicode_AsUTF8( pyname ); @@ -59,45 +57,46 @@ static PyObject * PyCspEnumMeta_new( PyTypeObject *subtype, PyObject *args, PyOb CSP_THROW( PythonPassthrough, "" ); if( !PyLong_Check( value ) ) - CSP_THROW( TypeError, "csp.Enum key " << keystr << " expected an integer got " << PyObjectPtr::incref( value ) ); + CSP_THROW( TypeError, + "csp.Enum key " << keystr << " expected an integer got " << PyObjectPtr::incref( value ) ); - def[ keystr ] = fromPython( value ); + def[keystr] = fromPython( value ); } } - //back reference to the csp enum type that will be accessible on the csp enum -> meta() - //intentionally dont incref here to break the circular dep of type -> shared_ptr on CspEnumMeta - PyTypeObjectPtr typePtr = PyTypeObjectPtr::own( ( PyTypeObject * ) pymeta ); - auto enumMeta = std::make_shared( typePtr, name, def ); + // back reference to the csp enum type that will be accessible on the csp enum -> meta() + // intentionally dont incref here to break the circular dep of type -> shared_ptr on CspEnumMeta + PyTypeObjectPtr typePtr = PyTypeObjectPtr::own( (PyTypeObject *)pymeta ); + auto enumMeta = std::make_shared( typePtr, name, def ); - pymeta -> enumMeta = enumMeta; + pymeta->enumMeta = enumMeta; - //pre-create instances - pymeta -> enumsByName = PyObjectPtr::own( PyDict_New() ); - pymeta -> enumsByValue = PyObjectPtr::own( PyDict_New() ); + // pre-create instances + pymeta->enumsByName = PyObjectPtr::own( PyDict_New() ); + pymeta->enumsByValue = PyObjectPtr::own( PyDict_New() ); - for( auto & [ key, value ] : def ) + for( auto & [key, value] : def ) { - PyCspEnum * enum_ = ( PyCspEnum * ) ( (PyTypeObject * ) pymeta ) -> tp_alloc( (PyTypeObject * ) pymeta, 0 ); + PyCspEnum * enum_ = (PyCspEnum *)( (PyTypeObject *)pymeta )->tp_alloc( (PyTypeObject *)pymeta, 0 ); - new( enum_ ) PyCspEnum( enumMeta -> create( value ) ); - enum_ -> enumName = PyObjectPtr::own( toPython( key ) ); - enum_ -> enumValue = PyObjectPtr::own( toPython( value ) ); + new( enum_ ) PyCspEnum( enumMeta->create( value ) ); + enum_->enumName = PyObjectPtr::own( toPython( key ) ); + enum_->enumValue = PyObjectPtr::own( toPython( value ) ); - pymeta -> enumsByCValue[ value ] = PyObjectPtr::incref( enum_ ); + pymeta->enumsByCValue[value] = PyObjectPtr::incref( enum_ ); - if( PyDict_SetItem( pymeta -> enumsByName.get(), enum_ -> enumName.get(), enum_ ) < 0 ) + if( PyDict_SetItem( pymeta->enumsByName.get(), enum_->enumName.get(), enum_ ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItem( pymeta -> enumsByValue.get(), enum_ -> enumValue.get(), enum_ ) < 0 ) + if( PyDict_SetItem( pymeta->enumsByValue.get(), enum_->enumValue.get(), enum_ ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - //We also have to update the items in the actual type's dict so FooEnum.A is a PyCspEnum! - if( PyDict_SetItem( ( ( PyTypeObject * ) pymeta ) -> tp_dict, enum_ -> enumName.get(), enum_ ) < 0 ) + // We also have to update the items in the actual type's dict so FooEnum.A is a PyCspEnum! + if( PyDict_SetItem( ( (PyTypeObject *)pymeta )->tp_dict, enum_->enumName.get(), enum_ ) < 0 ) CSP_THROW( PythonPassthrough, "" ); } - return ( PyObject * ) pymeta; + return (PyObject *)pymeta; CSP_RETURN_NULL; } @@ -107,90 +106,88 @@ PyObject * PyCspEnumMeta::toPyEnum( CspEnum e ) const if( it == enumsByCValue.end() ) return nullptr; - PyObject * rv = it -> second.get(); + PyObject * rv = it->second.get(); Py_INCREF( rv ); return rv; } void PyCspEnumMeta_dealloc( PyCspEnumMeta * m ) { - CspTypeFactory::instance().removeCachedType( reinterpret_cast( m ) ); - m -> ~PyCspEnumMeta(); - Py_TYPE( m ) -> tp_free( m ); + CspTypeFactory::instance().removeCachedType( reinterpret_cast( m ) ); + m->~PyCspEnumMeta(); + Py_TYPE( m )->tp_free( m ); } PyObject * PyCspEnumMeta_subscript( PyCspEnumMeta * self, PyObject * key ) { CSP_BEGIN_METHOD; - - PyObject * obj = PyDict_GetItem( self -> enumsByName.get(), key ); + + PyObject * obj = PyDict_GetItem( self->enumsByName.get(), key ); if( !obj ) - CSP_THROW( ValueError, PyObjectPtr::incref( key ) << " is not a valid name on csp.enum type " << ( ( PyTypeObject * ) self ) -> tp_name ); + CSP_THROW( ValueError, + PyObjectPtr::incref( key ) + << " is not a valid name on csp.enum type " << ( (PyTypeObject *)self )->tp_name ); Py_INCREF( obj ); return obj; CSP_RETURN_NULL; } - static PyMappingMethods PyCspEnumMeta_MappingMethods = { - 0, /*mp_length */ - (binaryfunc) PyCspEnumMeta_subscript, /*mp_subscript */ + 0, /*mp_length */ + (binaryfunc)PyCspEnumMeta_subscript, /*mp_subscript */ }; PyTypeObject PyCspEnumMeta::PyType = { - PyVarObject_HEAD_INIT(nullptr, 0) - "_cspimpl.PyCspEnumMeta", /* tp_name */ - sizeof(PyCspEnumMeta), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) PyCspEnumMeta_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - &PyCspEnumMeta_MappingMethods, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS, /* tp_flags */ - "csp enum metaclass", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyType_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /*tp_init*/ - 0, /* tp_alloc */ - (newfunc) PyCspEnumMeta_new,/* tp_new */ - PyObject_GC_Del, /* tp_free */ + PyVarObject_HEAD_INIT( nullptr, 0 ) "_cspimpl.PyCspEnumMeta", /* tp_name */ + sizeof( PyCspEnumMeta ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyCspEnumMeta_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + &PyCspEnumMeta_MappingMethods, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS, /* tp_flags */ + "csp enum metaclass", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyType_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /*tp_init*/ + 0, /* tp_alloc */ + (newfunc)PyCspEnumMeta_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ }; - -//PyCspEnum +// PyCspEnum void PyCspEnum_dealloc( PyCspEnum * self ) { - self -> ~PyCspEnum(); - Py_TYPE( self ) -> tp_free( self ); + self->~PyCspEnum(); + Py_TYPE( self )->tp_free( self ); } -PyObject * PyCspEnum_new( PyTypeObject * type, PyObject *args, PyObject *kwds ) +PyObject * PyCspEnum_new( PyTypeObject * type, PyObject * args, PyObject * kwds ) { CSP_BEGIN_METHOD; @@ -198,15 +195,16 @@ PyObject * PyCspEnum_new( PyTypeObject * type, PyObject *args, PyObject *kwds ) if( !PyArg_ParseTuple( args, "O", &pyvalue ) ) CSP_THROW( PythonPassthrough, "" ); - auto pymeta = (PyCspEnumMeta * ) type; + auto pymeta = (PyCspEnumMeta *)type; PyObject * obj = nullptr; if( PyLong_Check( pyvalue ) ) - obj = PyDict_GetItem( pymeta -> enumsByValue.get(), pyvalue ); + obj = PyDict_GetItem( pymeta->enumsByValue.get(), pyvalue ); else if( PyUnicode_Check( pyvalue ) ) - obj = PyDict_GetItem( pymeta -> enumsByName.get(), pyvalue ); + obj = PyDict_GetItem( pymeta->enumsByName.get(), pyvalue ); if( !obj ) - CSP_THROW( ValueError, PyObjectPtr::incref( pyvalue ) << " is not a valid value on csp.enum type " << type -> tp_name ); + CSP_THROW( ValueError, + PyObjectPtr::incref( pyvalue ) << " is not a valid value on csp.enum type " << type->tp_name ); Py_INCREF( obj ); return obj; @@ -215,66 +213,63 @@ PyObject * PyCspEnum_new( PyTypeObject * type, PyObject *args, PyObject *kwds ) PyObject * PyCspEnum_name( PyCspEnum * self, void * ) { - Py_INCREF( self -> enumName.get() ); - return self -> enumName.get(); + Py_INCREF( self->enumName.get() ); + return self->enumName.get(); } PyObject * PyCspEnum_value( PyCspEnum * self, void * ) { - Py_INCREF( self -> enumValue.get() ); - return self -> enumValue.get(); + Py_INCREF( self->enumValue.get() ); + return self->enumValue.get(); } -static PyGetSetDef PyCspEnum_getset[] = { - { ( char * ) "name", (getter) PyCspEnum_name, 0, ( char * ) "string name of the enum instance", 0 }, - { ( char * ) "value", (getter) PyCspEnum_value, 0, ( char * ) "long value of the enum instance", 0 }, - { NULL } -}; +static PyGetSetDef PyCspEnum_getset[] + = { { (char *)"name", (getter)PyCspEnum_name, 0, (char *)"string name of the enum instance", 0 }, + { (char *)"value", (getter)PyCspEnum_value, 0, (char *)"long value of the enum instance", 0 }, + { NULL } }; PyTypeObject PyCspEnum::PyType = { - PyVarObject_HEAD_INIT(nullptr, 0) - "_cspimpl.PyCspEnum", /* tp_name */ - sizeof(PyCspEnum), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) PyCspEnum_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - "csp enum", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - PyCspEnum_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - (newfunc) PyCspEnum_new, /* tp_new */ - 0, /* tp_free */ + PyVarObject_HEAD_INIT( nullptr, 0 ) "_cspimpl.PyCspEnum", /* tp_name */ + sizeof( PyCspEnum ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyCspEnum_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + "csp enum", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + PyCspEnum_getset, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + (newfunc)PyCspEnum_new, /* tp_new */ + 0, /* tp_free */ }; REGISTER_TYPE_INIT( &PyCspEnumMeta::PyType, "PyCspEnumMeta" ) -REGISTER_TYPE_INIT( &PyCspEnum::PyType, "PyCspEnum" ) +REGISTER_TYPE_INIT( &PyCspEnum::PyType, "PyCspEnum" ) -} +} // namespace csp::python diff --git a/cpp/csp/python/PyCspType.cpp b/cpp/csp/python/PyCspType.cpp index 02ff8b4a..448a8fb7 100644 --- a/cpp/csp/python/PyCspType.cpp +++ b/cpp/csp/python/PyCspType.cpp @@ -1,6 +1,6 @@ +#include #include #include -#include #include static_assert( sizeof( csp::DialectGenericType ) == sizeof( csp::python::PyObjectPtr ) ); @@ -8,56 +8,52 @@ static_assert( alignof( csp::DialectGenericType ) == alignof( csp::python::PyObj namespace csp { -DialectGenericType::DialectGenericType() -{ - new( this ) csp::python::PyObjectPtr(); -} +DialectGenericType::DialectGenericType() { new( this ) csp::python::PyObjectPtr(); } DialectGenericType::~DialectGenericType() { using T = csp::python::PyObjectPtr; - reinterpret_cast(this) -> ~T(); + reinterpret_cast( this )->~T(); } -DialectGenericType::DialectGenericType( const DialectGenericType &rhs ) +DialectGenericType::DialectGenericType( const DialectGenericType & rhs ) { - new( this ) csp::python::PyObjectPtr( reinterpret_cast(rhs) ); + new( this ) csp::python::PyObjectPtr( reinterpret_cast( rhs ) ); } -DialectGenericType::DialectGenericType( DialectGenericType &&rhs ) +DialectGenericType::DialectGenericType( DialectGenericType && rhs ) { - new( this ) csp::python::PyObjectPtr( reinterpret_cast(rhs) ); + new( this ) csp::python::PyObjectPtr( reinterpret_cast( rhs ) ); } DialectGenericType DialectGenericType::deepcopy() const { static PyObject * pyDeepcopy = PyObject_GetAttrString( PyImport_ImportModule( "copy" ), "deepcopy" ); - PyObject * pyVal = PyObject_CallFunction( pyDeepcopy, "(O)", python::toPythonBorrowed( *this ) ); - return DialectGenericType( reinterpret_cast( std::move( csp::python::PyObjectPtr::check( pyVal ) ) ) ); + PyObject * pyVal = PyObject_CallFunction( pyDeepcopy, "(O)", python::toPythonBorrowed( *this ) ); + return DialectGenericType( + reinterpret_cast( std::move( csp::python::PyObjectPtr::check( pyVal ) ) ) ); } -DialectGenericType &DialectGenericType::operator=( const DialectGenericType &rhs ) +DialectGenericType & DialectGenericType::operator=( const DialectGenericType & rhs ) { - *reinterpret_cast(this) = reinterpret_cast(rhs); + *reinterpret_cast( this ) = reinterpret_cast( rhs ); return *this; } -DialectGenericType &DialectGenericType::operator=( DialectGenericType &&rhs ) +DialectGenericType & DialectGenericType::operator=( DialectGenericType && rhs ) { - *reinterpret_cast(this) = std::move( reinterpret_cast(rhs) ); + *reinterpret_cast( this ) + = std::move( reinterpret_cast( rhs ) ); return *this; } - -bool DialectGenericType::operator==( const DialectGenericType &rhs ) const +bool DialectGenericType::operator==( const DialectGenericType & rhs ) const { - return *reinterpret_cast(this) == reinterpret_cast(rhs); + return *reinterpret_cast( this ) + == reinterpret_cast( rhs ); } -size_t DialectGenericType::hash() const -{ - return reinterpret_cast(this) -> hash(); -} +size_t DialectGenericType::hash() const { return reinterpret_cast( this )->hash(); } std::ostream & operator<<( std::ostream & o, const DialectGenericType & obj ) { @@ -65,4 +61,4 @@ std::ostream & operator<<( std::ostream & o, const DialectGenericType & obj ) return o; } -} +} // namespace csp diff --git a/cpp/csp/python/PyDynamicNode.cpp b/cpp/csp/python/PyDynamicNode.cpp index c38aff7c..8a519efb 100644 --- a/cpp/csp/python/PyDynamicNode.cpp +++ b/cpp/csp/python/PyDynamicNode.cpp @@ -5,8 +5,8 @@ #include #include #include -#include #include +#include #include namespace csp::python @@ -24,36 +24,31 @@ static PyObject * PyDynamicNode_create( PyObject * module, PyObject * args ) PyObject * pyDynamicArgs; PyObject * pySubgraphArgs; - if( !PyArg_ParseTuple( args, "O!sO!O!OO!O!", - &PyEngine::PyType, &pyEngine, - &name, - &PyTuple_Type, &inputs, - &PyTuple_Type, &outputs, - &pyBuilder, - &PyTuple_Type, &pyDynamicArgs, - &PyList_Type, &pySubgraphArgs) ) + if( !PyArg_ParseTuple( args, "O!sO!O!OO!O!", &PyEngine::PyType, &pyEngine, &name, &PyTuple_Type, &inputs, + &PyTuple_Type, &outputs, &pyBuilder, &PyTuple_Type, &pyDynamicArgs, &PyList_Type, + &pySubgraphArgs ) ) CSP_THROW( PythonPassthrough, "" ); PyObjectPtr pySubgraphArgsObj = PyObjectPtr::incref( pySubgraphArgs ); - PyObjectPtr pyBuilderObj = PyObjectPtr::incref( pyBuilder ); - PyObjectPtr pyOutputsObj = PyObjectPtr::incref( outputs ); + PyObjectPtr pyBuilderObj = PyObjectPtr::incref( pyBuilder ); + PyObjectPtr pyOutputsObj = PyObjectPtr::incref( outputs ); - //vector of ts input index -> arg location to snap into - std::vector> snapMap; - std::vector snapIds; //used for makepassive call + // vector of ts input index -> arg location to snap into + std::vector> snapMap; + std::vector snapIds; // used for makepassive call - //args that requested snapkey + // args that requested snapkey std::vector snapkeyArgs; - //same for attach, even though we currently only allow ataching to trigger input 0 - std::vector> attachMap; + // same for attach, even though we currently only allow ataching to trigger input 0 + std::vector> attachMap; int len = PyTuple_GET_SIZE( pyDynamicArgs ); for( int i = 0; i < len; ++i ) { - PyObject * item = PyTuple_GET_ITEM( pyDynamicArgs, i ); - auto ts_idx = fromPython( PyTuple_GET_ITEM( item, 0 ) ); - auto arg_idx = fromPython( PyTuple_GET_ITEM( item, 1 ) ); + PyObject * item = PyTuple_GET_ITEM( pyDynamicArgs, i ); + auto ts_idx = fromPython( PyTuple_GET_ITEM( item, 0 ) ); + auto arg_idx = fromPython( PyTuple_GET_ITEM( item, 1 ) ); const char * arg_type = PyUnicode_AsUTF8( PyTuple_GET_ITEM( item, 2 ) ); if( strcmp( arg_type, "snap" ) == 0 ) { @@ -68,75 +63,77 @@ static PyObject * PyDynamicNode_create( PyObject * module, PyObject * args ) CSP_THROW( ValueError, "Internal Error: unrecognized dynamic arg type: " << arg_type ); } - DynamicNode::EngineBuilder builder = [pyBuilderObj, - snapMap = std::move( snapMap ), - attachMap = std::move( attachMap ), - snapkeyArgs = std::move( snapkeyArgs ), - pySubgraphArgsObj, - pyOutputsObj ]( DynamicNode * pNode, DynamicEngine * engine, const DialectGenericType & newkey ) + DynamicNode::EngineBuilder builder + = [pyBuilderObj, snapMap = std::move( snapMap ), attachMap = std::move( attachMap ), + snapkeyArgs = std::move( snapkeyArgs ), pySubgraphArgsObj, + pyOutputsObj]( DynamicNode * pNode, DynamicEngine * engine, const DialectGenericType & newkey ) + { + // this is the point where we adjust the "raw args" for any dynamic args + + // csp.snap + for( auto & entry : snapMap ) + { + auto ts_idx = entry.first; + auto arg_idx = entry.second; + + // TODO support basket snap ? + auto * ts = pNode->tsinput( ts_idx ); + if( !ts->valid() ) + CSP_THROW( RuntimeException, + "csp.snap input ( sub_graph arg " + << arg_idx << " ) is not valid at time of dynamic creation on csp.dynamic node '" + << pNode->name() << "'" ); + auto * pyArg = lastValueToPython( ts ); + + // decref previous arg + Py_DECREF( PyList_GET_ITEM( pySubgraphArgsObj.get(), arg_idx ) ); + PyList_SET_ITEM( pySubgraphArgsObj.get(), arg_idx, pyArg ); + } + + // csp.snapkey + for( auto & arg_idx : snapkeyArgs ) + { + // we only allow snapkey on the trigger basket, so just pass along new key + auto * pykey = toPython( newkey ); + Py_DECREF( PyList_GET_ITEM( pySubgraphArgsObj.get(), arg_idx ) ); + PyList_SET_ITEM( pySubgraphArgsObj.get(), arg_idx, pykey ); + } + + // csp.attach + for( auto & entry : attachMap ) { - //this is the point where we adjust the "raw args" for any dynamic args - - //csp.snap - for( auto & entry : snapMap ) - { - auto ts_idx = entry.first; - auto arg_idx = entry.second; - - //TODO support basket snap ? - auto * ts = pNode -> tsinput( ts_idx ); - if( !ts -> valid() ) - CSP_THROW( RuntimeException, "csp.snap input ( sub_graph arg " << arg_idx << " ) is not valid at time of dynamic creation on csp.dynamic node '" << pNode -> name() << "'" ); - auto * pyArg = lastValueToPython( ts ); - - //decref previous arg - Py_DECREF( PyList_GET_ITEM( pySubgraphArgsObj.get(), arg_idx ) ); - PyList_SET_ITEM( pySubgraphArgsObj.get(), arg_idx, pyArg ); - } - - //csp.snapkey - for( auto & arg_idx : snapkeyArgs ) - { - //we only allow snapkey on the trigger basket, so just pass along new key - auto * pykey = toPython( newkey ); - Py_DECREF( PyList_GET_ITEM( pySubgraphArgsObj.get(), arg_idx ) ); - PyList_SET_ITEM( pySubgraphArgsObj.get(), arg_idx, pykey ); - } - - //csp.attach - for( auto & entry : attachMap ) - { - auto arg_idx = entry.second; - auto edge_id = pNode -> elemId( newkey ); - - PyObject * edge = PyList_GET_ITEM( pySubgraphArgsObj.get(), arg_idx ); - - //python side wiring already setup the Edge for csp.attach, just need to update the elemId properly - if( PyObject_SetAttrString( edge, "basket_idx", toPython( edge_id ) ) < 0 ) - CSP_THROW( PythonPassthrough, "" ); - } - - //temporary python wrapper for graph building - PyPtr pyEngineWrapper = PyPtr::own( PyEngine::create( engine ) ); - PyObjectPtr rv = PyObjectPtr::check( PyObject_CallFunctionObjArgs( pyBuilderObj.get(), pyEngineWrapper.get(), pySubgraphArgsObj.get(), NULL ) ); - - //collect timeseries that need to be fed back out over dynamic basket outputs - DynamicNode::Outputs outputs; - Py_ssize_t numOutputs = PyTuple_GET_SIZE( pyOutputsObj.get() ); - for( auto i = 0; i < numOutputs; ++i ) - { - PyObject * key = PyTuple_GET_ITEM( pyOutputsObj.get(), i ); - outputs.emplace_back( engine -> outputTs( fromPython( key ) ) ); - } - - return outputs; - }; - - Py_ssize_t numInputs = PyTuple_GET_SIZE( inputs ); + auto arg_idx = entry.second; + auto edge_id = pNode->elemId( newkey ); + + PyObject * edge = PyList_GET_ITEM( pySubgraphArgsObj.get(), arg_idx ); + + // python side wiring already setup the Edge for csp.attach, just need to update the elemId properly + if( PyObject_SetAttrString( edge, "basket_idx", toPython( edge_id ) ) < 0 ) + CSP_THROW( PythonPassthrough, "" ); + } + + // temporary python wrapper for graph building + PyPtr pyEngineWrapper = PyPtr::own( PyEngine::create( engine ) ); + PyObjectPtr rv = PyObjectPtr::check( + PyObject_CallFunctionObjArgs( pyBuilderObj.get(), pyEngineWrapper.get(), pySubgraphArgsObj.get(), NULL ) ); + + // collect timeseries that need to be fed back out over dynamic basket outputs + DynamicNode::Outputs outputs; + Py_ssize_t numOutputs = PyTuple_GET_SIZE( pyOutputsObj.get() ); + for( auto i = 0; i < numOutputs; ++i ) + { + PyObject * key = PyTuple_GET_ITEM( pyOutputsObj.get(), i ); + outputs.emplace_back( engine->outputTs( fromPython( key ) ) ); + } + + return outputs; + }; + + Py_ssize_t numInputs = PyTuple_GET_SIZE( inputs ); Py_ssize_t numOutputs = PyTuple_GET_SIZE( outputs ); NodeDef def( numInputs, numOutputs ); - DynamicNode * node = pyEngine -> engine() -> createOwnedObject( name, snapIds, builder, def ); + DynamicNode * node = pyEngine->engine()->createOwnedObject( name, snapIds, builder, def ); return PyNodeWrapper::create( node ); CSP_RETURN_NULL; @@ -144,4 +141,4 @@ static PyObject * PyDynamicNode_create( PyObject * module, PyObject * args ) REGISTER_MODULE_METHOD( "PyDynamicNode", PyDynamicNode_create, METH_VARARGS, "PyDynamicNode" ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyEngine.cpp b/cpp/csp/python/PyEngine.cpp index e6f90de0..070e31ce 100644 --- a/cpp/csp/python/PyEngine.cpp +++ b/cpp/csp/python/PyEngine.cpp @@ -6,17 +6,17 @@ #include #include - -//define engine-level csp switch types here +// define engine-level csp switch types here #include namespace csp::python { -//The actual engine object -PythonEngine::PythonEngine( PyEngine * parent, const Dictionary & settings ) : RootEngine( settings ), - m_parent( parent ), - m_pyThreadState( nullptr ) +// The actual engine object +PythonEngine::PythonEngine( PyEngine * parent, const Dictionary & settings ) + : RootEngine( settings ) + , m_parent( parent ) + , m_pyThreadState( nullptr ) { m_outputNumpy = settings.get( "output_numpy", false ); } @@ -33,17 +33,17 @@ void PythonEngine::dialectLockGIL() noexcept m_pyThreadState = nullptr; } -//The engine python wrapper object +// The engine python wrapper object PyEngine::PyEngine( const Dictionary & settings ) { m_ownEngine = true; - m_engine = new PythonEngine( this, settings ); + m_engine = new PythonEngine( this, settings ); } PyEngine::PyEngine( Engine * engine ) { m_ownEngine = false; - m_engine = engine; + m_engine = engine; } PyEngine::~PyEngine() @@ -57,18 +57,18 @@ PyObject * PyEngine::collectOutputs() { CSP_BEGIN_METHOD; - if( m_engine -> graphOutputKeys().empty() || rootEngine() -> interrupted() ) + if( m_engine->graphOutputKeys().empty() || rootEngine()->interrupted() ) Py_RETURN_NONE; PyObjectPtr out = PyObjectPtr::own( PyDict_New() ); - for( auto & key : m_engine -> graphOutputKeys() ) + for( auto & key : m_engine->graphOutputKeys() ) { - //Not sure if there is a more efficient way to do this other than converting per element ( aside from using numpy arrays which would - //impose a dep on numpy ) - auto * adapter = static_cast( m_engine -> graphOutput( key ) ); - auto & pykey = reinterpret_cast( key ); - if( PyDict_SetItem( out.ptr(), pykey.ptr(), adapter -> result().ptr() ) < 0 ) + // Not sure if there is a more efficient way to do this other than converting per element ( aside from using + // numpy arrays which would impose a dep on numpy ) + auto * adapter = static_cast( m_engine->graphOutput( key ) ); + auto & pykey = reinterpret_cast( key ); + if( PyDict_SetItem( out.ptr(), pykey.ptr(), adapter->result().ptr() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); } @@ -78,10 +78,9 @@ PyObject * PyEngine::collectOutputs() PyEngine * PyEngine::create( Engine * engine ) { - PyEngine * pyengine = ( PyEngine * ) PyType.tp_new( &PyType, nullptr, nullptr ); - new ( pyengine ) PyEngine( engine ); + PyEngine * pyengine = (PyEngine *)PyType.tp_new( &PyType, nullptr, nullptr ); + new( pyengine ) PyEngine( engine ); return pyengine; - } static int PyEngine_init( PyEngine * self, PyObject * args, PyObject * kwargs ) @@ -94,8 +93,8 @@ static int PyEngine_init( PyEngine * self, PyObject * args, PyObject * kwargs ) static void PyEngine_dealloc( PyEngine * self ) { CSP_BEGIN_METHOD; - self -> ~PyEngine(); - Py_TYPE( self ) -> tp_free( self ); + self->~PyEngine(); + Py_TYPE( self )->tp_free( self ); CSP_RETURN; } @@ -105,75 +104,75 @@ static PyObject * PyEngine_run( PyEngine * self, PyObject * args ) PyObject * pyStart; PyObject * pyEnd; - if( !PyArg_ParseTuple( args, "OO", &pyStart, &pyEnd ) ) + if( !PyArg_ParseTuple( args, "OO", &pyStart, &pyEnd ) ) return nullptr; auto start = fromPython( pyStart ); - auto end = fromPython( pyEnd); + auto end = fromPython( pyEnd ); - CSP_TRUE_OR_THROW_RUNTIME( self -> engine() -> isRootEngine(), "engine is not root engine" ); - self -> rootEngine() -> run( start, end ); + CSP_TRUE_OR_THROW_RUNTIME( self->engine()->isRootEngine(), "engine is not root engine" ); + self->rootEngine()->run( start, end ); - return self -> collectOutputs(); + return self->collectOutputs(); CSP_RETURN_NONE; } -static PyMethodDef PyEngine_methods[] = { - { "run", (PyCFunction) PyEngine_run, METH_VARARGS, "start and run engine" }, - { NULL } -}; +static PyMethodDef PyEngine_methods[] + = { { "run", (PyCFunction)PyEngine_run, METH_VARARGS, "start and run engine" }, { NULL } }; PyTypeObject PyEngine::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyEngine", /* tp_name */ - sizeof(PyEngine), /* tp_basicsize */ - 0, /* tp_itemsize */ - ( destructor ) PyEngine_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp engine", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyEngine_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - ( initproc )PyEngine_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ /* Low-level free-memory routine */ - 0, /* tp_is_gc */ /* For PyObject_IS_GC */ - 0, /* tp_bases */ - 0, /* tp_mro */ /* method resolution order */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0 /* tp_finalize */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyEngine", /* tp_name */ + sizeof( PyEngine ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyEngine_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp engine", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyEngine_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)PyEngine_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + 0, + /* tp_free */ /* Low-level free-memory routine */ + 0, + /* tp_is_gc */ /* For PyObject_IS_GC */ + 0, /* tp_bases */ + 0, + /* tp_mro */ /* method resolution order */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0 /* tp_finalize */ }; REGISTER_TYPE_INIT( &PyEngine::PyType, "PyEngine" ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyFeedbackAdapter.cpp b/cpp/csp/python/PyFeedbackAdapter.cpp index 242ddcb4..360cf839 100644 --- a/cpp/csp/python/PyFeedbackAdapter.cpp +++ b/cpp/csp/python/PyFeedbackAdapter.cpp @@ -11,35 +11,33 @@ static OutputAdapter * output_creator( csp::AdapterManager * manager, PyEngine * { PyObject * pyType; PyInputAdapterWrapper * pyFeedbackInput = nullptr; - - if( !PyArg_ParseTuple( args, "OO!", - &pyType, - &PyInputAdapterWrapper::PyType, &pyFeedbackInput ) ) + + if( !PyArg_ParseTuple( args, "OO!", &pyType, &PyInputAdapterWrapper::PyType, &pyFeedbackInput ) ) CSP_THROW( PythonPassthrough, "" ); auto & cspType = pyTypeAsCspType( pyType ); return switchCspType( cspType, - [ pyengine, pyFeedbackInput ]( auto tag ) -> OutputAdapter * + [pyengine, pyFeedbackInput]( auto tag ) -> OutputAdapter * { - using T = typename decltype(tag)::type; - return pyengine -> engine() - -> createOwnedObject>( pyFeedbackInput -> adapter() ); + using T = typename decltype( tag )::type; + return pyengine->engine()->createOwnedObject>( + pyFeedbackInput->adapter() ); } ); } -static InputAdapter * input_creator( csp::AdapterManager * manager, PyEngine * engine, - PyObject * pyType, PushMode pushMode, PyObject * args ) +static InputAdapter * input_creator( csp::AdapterManager * manager, PyEngine * engine, PyObject * pyType, + PushMode pushMode, PyObject * args ) { auto & cspType = pyTypeAsCspType( pyType ); return switchCspType( cspType.get(), - [ engine, &cspType, pushMode ]( auto tag ) -> InputAdapter * + [engine, &cspType, pushMode]( auto tag ) -> InputAdapter * { - using T = typename decltype(tag)::type; - return engine -> engine() -> createOwnedObject>( cspType, pushMode ); + using T = typename decltype( tag )::type; + return engine->engine()->createOwnedObject>( cspType, pushMode ); } ); } REGISTER_INPUT_ADAPTER( _feedback_input_adapter, input_creator ); REGISTER_OUTPUT_ADAPTER( _feedback_output_adapter, output_creator ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyGraphOutputAdapter.cpp b/cpp/csp/python/PyGraphOutputAdapter.cpp index e3ba7d11..63dd34c6 100644 --- a/cpp/csp/python/PyGraphOutputAdapter.cpp +++ b/cpp/csp/python/PyGraphOutputAdapter.cpp @@ -1,10 +1,10 @@ #include #include +#include +#include #include #include -#include #include -#include namespace csp::python { @@ -20,73 +20,71 @@ PyObjectPtr PyGraphOutputAdapter::result() void PyGraphOutputAdapter::processResults() { auto * ts = input(); - auto len = tickCount() > 0 ? std::min( tickCount(), ts -> numTicks() ) : ts -> numTicks(); - //NOTE we may want to limit ticks to requested history as well... TBD? + auto len = tickCount() > 0 ? std::min( tickCount(), ts->numTicks() ) : ts->numTicks(); + // NOTE we may want to limit ticks to requested history as well... TBD? - if( static_cast( rootEngine() ) -> outputNumpy() ) + if( static_cast( rootEngine() )->outputNumpy() ) { - auto res = valuesAtIndexToNumpy( ValueType::TIMESTAMP_VALUE_TUPLE, - ts, len - 1, 0, autogen::TimeIndexPolicy::INCLUSIVE, autogen::TimeIndexPolicy::INCLUSIVE ); + auto res = valuesAtIndexToNumpy( ValueType::TIMESTAMP_VALUE_TUPLE, ts, len - 1, 0, + autogen::TimeIndexPolicy::INCLUSIVE, autogen::TimeIndexPolicy::INCLUSIVE ); m_result = PyObjectPtr::check( res ); } else { m_result = PyObjectPtr::check( PyList_New( len ) ); - switchCspType( ts -> type(), - [ this, ts, len ]( auto tag ) + switchCspType( ts->type(), + [this, ts, len]( auto tag ) { - Py_ssize_t idx = 0; + Py_ssize_t idx = 0; for( int32_t index = len - 1; index >= 0; --index ) { - PyObjectPtr pydt = PyObjectPtr::own( toPython( ts -> timeAtIndex( index ) ) ); + PyObjectPtr pydt = PyObjectPtr::own( toPython( ts->timeAtIndex( index ) ) ); PyObjectPtr pyvalue = PyObjectPtr::own( - toPython( ts -> valueAtIndex( index ), *ts -> type() ) ); - PyObjectPtr entry = PyObjectPtr::check( PyTuple_Pack( 2, pydt.ptr(), pyvalue.ptr() ) ); + toPython( ts->valueAtIndex( index ), *ts->type() ) ); + PyObjectPtr entry = PyObjectPtr::check( PyTuple_Pack( 2, pydt.ptr(), pyvalue.ptr() ) ); PyList_SET_ITEM( m_result.ptr(), idx++, entry.release() ); } } ); } } -//csp.add_graph_output +// csp.add_graph_output static OutputAdapter * creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * args ) { - PyObject * pyKey = nullptr; - int tickCount = -1; + PyObject * pyKey = nullptr; + int tickCount = -1; PyObject * pyTickHistory = nullptr; - + if( !PyArg_ParseTuple( args, "OiO", &pyKey, &tickCount, &pyTickHistory ) ) CSP_THROW( PythonPassthrough, "" ); - auto * engine = pyengine -> engine(); - auto key = fromPython( pyKey ); - auto adapter = std::make_shared( engine, tickCount, fromPython( pyTickHistory ) ); - engine -> registerGraphOutput( key, adapter ); + auto * engine = pyengine->engine(); + auto key = fromPython( pyKey ); + auto adapter = std::make_shared( engine, tickCount, fromPython( pyTickHistory ) ); + engine->registerGraphOutput( key, adapter ); return adapter.get(); } -//for returning __outputs__ from a graph ( as opposed to csp.add_graph_output calls ) +// for returning __outputs__ from a graph ( as opposed to csp.add_graph_output calls ) static OutputAdapter * return_creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * args ) { - PyObject * pyKey = nullptr; + PyObject * pyKey = nullptr; if( !PyArg_ParseTuple( args, "O", &pyKey ) ) CSP_THROW( PythonPassthrough, "" ); - auto * engine = pyengine -> engine(); - if( engine -> isRootEngine() ) + auto * engine = pyengine->engine(); + if( engine->isRootEngine() ) { - PyObjectPtr argsEx = PyObjectPtr::check( Py_BuildValue( "OiN", - pyKey, - -1, - toPython( TimeDelta::NONE() ) ) ); + PyObjectPtr argsEx = PyObjectPtr::check( Py_BuildValue( "OiN", pyKey, -1, toPython( TimeDelta::NONE() ) ) ); return creator( manager, pyengine, argsEx.get() ); } - //dynamic graph output - auto * adapter = engine -> createOwnedObject(); + // dynamic graph output + auto * adapter = engine->createOwnedObject(); - //for graph outputs keys should always be strings, unnamed single should be "". For historical reasons unnamed is passed as 0 + // for graph outputs keys should always be strings, unnamed single should be "". For historical reasons unnamed is + // passed as 0 std::string key; if( PyUnicode_Check( pyKey ) ) key = fromPython( pyKey ); @@ -95,11 +93,11 @@ static OutputAdapter * return_creator( csp::AdapterManager * manager, PyEngine * assert( fromPython( pyKey ) == 0 ); } - static_cast( engine ) -> registerGraphOutput( key, adapter ); + static_cast( engine )->registerGraphOutput( key, adapter ); return adapter; } REGISTER_OUTPUT_ADAPTER( _graph_output_adapter, creator ); REGISTER_OUTPUT_ADAPTER( _graph_return_adapter, return_creator ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyInputAdapterWrapper.cpp b/cpp/csp/python/PyInputAdapterWrapper.cpp index 58384cc4..788413d3 100644 --- a/cpp/csp/python/PyInputAdapterWrapper.cpp +++ b/cpp/csp/python/PyInputAdapterWrapper.cpp @@ -2,9 +2,9 @@ #include #include #include +#include #include #include -#include namespace csp::python { @@ -13,26 +13,21 @@ PyObject * PyInputAdapterWrapper::createAdapter( Creator creator, PyObject * arg { CSP_BEGIN_METHOD; - PyObject * pyAdapterManager = nullptr; - PyEngine * pyEngine = nullptr; - PyObject * pyType = nullptr; - PyObject * pyArgs = nullptr; - int pushmode = -1; + PyEngine * pyEngine = nullptr; + PyObject * pyType = nullptr; + PyObject * pyArgs = nullptr; + int pushmode = -1; - if( !PyArg_ParseTuple( args, "OO!OiO!", - &pyAdapterManager, - &PyEngine::PyType, &pyEngine, - &pyType, - &pushmode, + if( !PyArg_ParseTuple( args, "OO!OiO!", &pyAdapterManager, &PyEngine::PyType, &pyEngine, &pyType, &pushmode, &PyTuple_Type, &pyArgs ) ) CSP_THROW( PythonPassthrough, "" ); if( pushmode == PushMode::UNKNOWN || pushmode >= PushMode::NUM_TYPES ) CSP_THROW( ValueError, "invalid pushmode " << pushmode ); - csp::AdapterManager *adapterMgr = nullptr; + csp::AdapterManager * adapterMgr = nullptr; if( PyCapsule_CheckExact( pyAdapterManager ) ) adapterMgr = PyAdapterManagerWrapper::extractAdapterManager( pyAdapterManager ); @@ -45,36 +40,36 @@ PyObject * PyInputAdapterWrapper::createAdapter( Creator creator, PyObject * arg PyObject * PyInputAdapterWrapper::create( InputAdapter * adapter ) { CSP_BEGIN_METHOD; - PyInputAdapterWrapper * object = ( PyInputAdapterWrapper * ) PyInputAdapterWrapper::PyType.tp_alloc( &PyInputAdapterWrapper::PyType, 0 ); + PyInputAdapterWrapper * object + = (PyInputAdapterWrapper *)PyInputAdapterWrapper::PyType.tp_alloc( &PyInputAdapterWrapper::PyType, 0 ); new( object ) PyInputAdapterWrapper( adapter ); return object; CSP_RETURN_NULL; } PyTypeObject PyInputAdapterWrapper::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyInputAdapterWrapper", /* tp_name */ - sizeof(PyInputAdapterWrapper), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp input adapter wrapper", /* tp_doc */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyInputAdapterWrapper", /* tp_name */ + sizeof( PyInputAdapterWrapper ), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp input adapter wrapper", /* tp_doc */ }; REGISTER_TYPE_INIT( &PyInputAdapterWrapper::PyType, "PyInputAdapterWrapper" ) -} +} // namespace csp::python diff --git a/cpp/csp/python/PyInputProxy.cpp b/cpp/csp/python/PyInputProxy.cpp index 656a126f..8661a926 100644 --- a/cpp/csp/python/PyInputProxy.cpp +++ b/cpp/csp/python/PyInputProxy.cpp @@ -1,73 +1,52 @@ +#include #include #include #include #include +#include #include #include -#include -#include - namespace csp::python { -PyInputProxy::PyInputProxy( PyNode * node, InputId id ) : m_node( node ), - m_id( id ) +PyInputProxy::PyInputProxy( PyNode * node, InputId id ) + : m_node( node ) + , m_id( id ) { } PyInputProxy * PyInputProxy::create( PyNode * node, InputId id ) { - PyInputProxy * proxy = ( PyInputProxy * ) PyType.tp_new( &PyType, nullptr, nullptr ); + PyInputProxy * proxy = (PyInputProxy *)PyType.tp_new( &PyType, nullptr, nullptr ); new( proxy ) PyInputProxy( node, id ); return proxy; } -TimeSeriesProvider * PyInputProxy::ts() const -{ - return const_cast( m_node -> input( m_id ) ); -} - -bool PyInputProxy::ticked() const -{ - return m_node -> inputTicked( m_id ); -} +TimeSeriesProvider * PyInputProxy::ts() const { return const_cast( m_node->input( m_id ) ); } -bool PyInputProxy::valid() const -{ - return ts() -> valid(); -} +bool PyInputProxy::ticked() const { return m_node->inputTicked( m_id ); } -uint32_t PyInputProxy::count() const -{ - return ts() -> count(); -} +bool PyInputProxy::valid() const { return ts()->valid(); } -uint32_t PyInputProxy::num_ticks() const -{ - return ts() -> numTicks(); -} +uint32_t PyInputProxy::count() const { return ts()->count(); } -bool PyInputProxy::makeActive() -{ - return m_node -> makeActive( m_id ); -} +uint32_t PyInputProxy::num_ticks() const { return ts()->numTicks(); } -bool PyInputProxy::makePassive() -{ - return m_node -> makePassive( m_id ); -} +bool PyInputProxy::makeActive() { return m_node->makeActive( m_id ); } + +bool PyInputProxy::makePassive() { return m_node->makePassive( m_id ); } Scheduler::Handle PyInputProxy::scheduleAlarm( DateTimeOrTimeDelta dtd, PyObject * value ) { - //TODO assert is alarm + // TODO assert is alarm auto * alarm = static_cast *>( const_cast( ts() ) ); - //as noted in PyNode, we always create alarms in PyNode as PyObjectPtr + // as noted in PyNode, we always create alarms in PyNode as PyObjectPtr if( dtd.isDateTime() ) - return alarm -> scheduleAlarm( dtd.datetime(), PyObjectPtr::incref( value ) ); + return alarm->scheduleAlarm( dtd.datetime(), PyObjectPtr::incref( value ) ); else - return alarm -> scheduleAlarm( dtd.timedelta(), PyObjectPtr::incref( value ) ); + return alarm->scheduleAlarm( dtd.timedelta(), PyObjectPtr::incref( value ) ); } Scheduler::Handle PyInputProxy::rescheduleAlarm( Scheduler::Handle handle, DateTimeOrTimeDelta dtd ) @@ -75,83 +54,89 @@ Scheduler::Handle PyInputProxy::rescheduleAlarm( Scheduler::Handle handle, DateT auto * alarm = static_cast *>( const_cast( ts() ) ); if( dtd.isDateTime() ) - return alarm -> rescheduleAlarm( handle, dtd.datetime() ); + return alarm->rescheduleAlarm( handle, dtd.datetime() ); else - return alarm -> rescheduleAlarm( handle, dtd.timedelta() ); + return alarm->rescheduleAlarm( handle, dtd.timedelta() ); } void PyInputProxy::cancelAlarm( Scheduler::Handle handle ) { auto * alarm = static_cast *>( const_cast( ts() ) ); - alarm -> cancelAlarm( handle ); + alarm->cancelAlarm( handle ); } void PyInputProxy::setBufferingPolicy( int32_t tickCount, TimeDelta tickHistory ) { if( tickCount > 0 ) - ts() -> setTickCountPolicy( tickCount ); - + ts()->setTickCountPolicy( tickCount ); + if( !tickHistory.isNone() && tickHistory > TimeDelta::ZERO() ) - ts() -> setTickTimeWindowPolicy( tickHistory ); + ts()->setTickTimeWindowPolicy( tickHistory ); } -PyObject *PyInputProxy::valueAt( ValueType valueType, PyObject *indexArg, PyObject *duplicatePolicyArg, PyObject *defaultValueArg ) const +PyObject * PyInputProxy::valueAt( ValueType valueType, PyObject * indexArg, PyObject * duplicatePolicyArg, + PyObject * defaultValueArg ) const { int32_t valueIndex; if( PyLong_Check( indexArg ) ) { - valueIndex = fromPython(indexArg); + valueIndex = fromPython( indexArg ); CSP_TRUE_OR_THROW_RUNTIME( valueIndex <= 0, "Expected non positive value for value_at index, got " << valueIndex ); // Convert negative value to actual positive index valueIndex = -valueIndex; - if( unlikely( valueIndex >= ts() -> tickCountPolicy() ) ) - CSP_THROW( RangeError, "buffer index out of range. requesting data at index " << valueIndex << " with buffer policy set to " << - ts() -> tickCountPolicy() << " ticks in node '" << m_node -> name() << "'" ); - + if( unlikely( valueIndex >= ts()->tickCountPolicy() ) ) + CSP_THROW( RangeError, + "buffer index out of range. requesting data at index " + << valueIndex << " with buffer policy set to " << ts()->tickCountPolicy() + << " ticks in node '" << m_node->name() << "'" ); } - else + else { - auto dtd = fromPython( indexArg ); + auto dtd = fromPython( indexArg ); auto duplicatePolicyIntVal = fromPython( duplicatePolicyArg ); - CSP_TRUE_OR_THROW_RUNTIME( duplicatePolicyIntVal == TimeSeriesProvider::DuplicatePolicyEnum::LAST_VALUE || - duplicatePolicyIntVal == TimeSeriesProvider::DuplicatePolicyEnum::FIRST_VALUE, + CSP_TRUE_OR_THROW_RUNTIME( duplicatePolicyIntVal == TimeSeriesProvider::DuplicatePolicyEnum::LAST_VALUE + || duplicatePolicyIntVal == TimeSeriesProvider::DuplicatePolicyEnum::FIRST_VALUE, "Unsupported duplicate policy " << duplicatePolicyIntVal ); - if( dtd.isTimeDelta() ) + if( dtd.isTimeDelta() ) { auto timedelta = dtd.timedelta(); - CSP_TRUE_OR_THROW_RUNTIME( !timedelta.isNone(), - "None time delta is unsupported" ); - CSP_TRUE_OR_THROW_RUNTIME( timedelta.sign() <= 0, - "Positive time delta is unsupported" ); - - if( unlikely( -timedelta > ts() -> tickTimeWindowPolicy() ) ) - CSP_THROW( RangeError, "buffer timedelta out of range. requesting data at timedelta " << PyObjectPtr::incref( indexArg ) << " with buffer policy set to " << - PyObjectPtr::own( toPython( ts() -> tickTimeWindowPolicy() ) ) << " in node '" << m_node -> name() << "'" ); - valueIndex = ts() -> getValueIndex( m_node -> now() + timedelta, duplicatePolicyIntVal ); + CSP_TRUE_OR_THROW_RUNTIME( !timedelta.isNone(), "None time delta is unsupported" ); + CSP_TRUE_OR_THROW_RUNTIME( timedelta.sign() <= 0, "Positive time delta is unsupported" ); + + if( unlikely( -timedelta > ts()->tickTimeWindowPolicy() ) ) + CSP_THROW( RangeError, + "buffer timedelta out of range. requesting data at timedelta " + << PyObjectPtr::incref( indexArg ) << " with buffer policy set to " + << PyObjectPtr::own( toPython( ts()->tickTimeWindowPolicy() ) ) << " in node '" + << m_node->name() << "'" ); + valueIndex = ts()->getValueIndex( m_node->now() + timedelta, duplicatePolicyIntVal ); } - else + else { CSP_TRUE_OR_THROW_RUNTIME( dtd.isDateTime(), "value_at index must be integer, DateTime, or TimeDelta" ); auto datetime = dtd.datetime(); - CSP_TRUE_OR_THROW_RUNTIME( datetime <= m_node -> now(), - "requesting data from future time" ); + CSP_TRUE_OR_THROW_RUNTIME( datetime <= m_node->now(), "requesting data from future time" ); - if( unlikely( m_node -> now() - datetime >= ts() -> tickTimeWindowPolicy() ) ) - CSP_THROW( RangeError, "requested buffer time out of range. requesting datetime " << PyObjectPtr::incref( indexArg ) << " at time " << - PyObjectPtr::own( toPython( m_node -> now() ) ) << " with buffer time window policy set to " << - PyObjectPtr::own( toPython( ts() -> tickTimeWindowPolicy() ) )<< " in node '" << m_node -> name() << "'" ); + if( unlikely( m_node->now() - datetime >= ts()->tickTimeWindowPolicy() ) ) + CSP_THROW( RangeError, + "requested buffer time out of range. requesting datetime " + << PyObjectPtr::incref( indexArg ) << " at time " + << PyObjectPtr::own( toPython( m_node->now() ) ) + << " with buffer time window policy set to " + << PyObjectPtr::own( toPython( ts()->tickTimeWindowPolicy() ) ) << " in node '" + << m_node->name() << "'" ); - valueIndex = ts() -> getValueIndex( datetime, duplicatePolicyIntVal ); + valueIndex = ts()->getValueIndex( datetime, duplicatePolicyIntVal ); } } - if( valueIndex < 0 || static_cast(valueIndex) >= num_ticks() ) + if( valueIndex < 0 || static_cast( valueIndex ) >= num_ticks() ) { if( defaultValueArg == constants::UNSET() ) CSP_THROW( OverflowError, "No matching value found" ); @@ -164,9 +149,10 @@ PyObject *PyInputProxy::valueAt( ValueType valueType, PyObject *indexArg, PyObje case ValueType::VALUE: return valueAtIndexToPython( ts(), valueIndex ); case ValueType::TIMESTAMP: - return toPython( ts() -> timeAtIndex( valueIndex ) ); + return toPython( ts()->timeAtIndex( valueIndex ) ); case ValueType::TIMESTAMP_VALUE_TUPLE: - return PyTuple_Pack( 2, toPython( ts() -> timeAtIndex( valueIndex ) ), valueAtIndexToPython( ts(), valueIndex ) ); + return PyTuple_Pack( 2, toPython( ts()->timeAtIndex( valueIndex ) ), + valueAtIndexToPython( ts(), valueIndex ) ); default: CSP_THROW( NotImplemented, "Unsupported value type " << valueType ); } @@ -179,33 +165,33 @@ int32_t PyInputProxy::computeStartIndex( DateTime startDt, autogen::TimeIndexPol switch( startPolicy.enum_value() ) { case autogen::TimeIndexPolicy::enum_::INCLUSIVE: - startIndex = ts() -> getValueIndex( startDt, TimeSeries::DuplicatePolicyEnum::FIRST_VALUE ); - if( startIndex != -1 && ts() -> timeAtIndex( startIndex ) < startDt ) + startIndex = ts()->getValueIndex( startDt, TimeSeries::DuplicatePolicyEnum::FIRST_VALUE ); + if( startIndex != -1 && ts()->timeAtIndex( startIndex ) < startDt ) { startIndex -= 1; - if( startIndex == -1 || ts() -> timeAtIndex( startIndex ) < startDt ) + if( startIndex == -1 || ts()->timeAtIndex( startIndex ) < startDt ) return -1; } break; case autogen::TimeIndexPolicy::enum_::EXCLUSIVE: - startIndex = ts() -> getValueIndex( startDt, TimeSeries::DuplicatePolicyEnum::LAST_VALUE ); - //if the timestamp matches, we need to exclude it - if( startIndex != -1 && ts() -> timeAtIndex( startIndex ) <= startDt ) + startIndex = ts()->getValueIndex( startDt, TimeSeries::DuplicatePolicyEnum::LAST_VALUE ); + // if the timestamp matches, we need to exclude it + if( startIndex != -1 && ts()->timeAtIndex( startIndex ) <= startDt ) { startIndex -= 1; - if( startIndex == -1 || ts() -> timeAtIndex( startIndex ) <= startDt ) + if( startIndex == -1 || ts()->timeAtIndex( startIndex ) <= startDt ) return -1; } break; case autogen::TimeIndexPolicy::enum_::EXTRAPOLATE: - startIndex = ts() -> getValueIndex( startDt, TimeSeries::DuplicatePolicyEnum::LAST_VALUE ); + startIndex = ts()->getValueIndex( startDt, TimeSeries::DuplicatePolicyEnum::LAST_VALUE ); break; default: CSP_THROW( InvalidArgument, "Unsupported time index policy " << startPolicy.name() ); } if( startIndex == -1 ) - startIndex = ts() -> numTicks() - 1; + startIndex = ts()->numTicks() - 1; return startIndex; } @@ -219,11 +205,11 @@ int32_t PyInputProxy::computeEndIndex( DateTime endDt, autogen::TimeIndexPolicy // for end index, inclusive is the same as forced case autogen::TimeIndexPolicy::enum_::INCLUSIVE: case autogen::TimeIndexPolicy::enum_::EXTRAPOLATE: - endIndex = ts() -> getValueIndex( endDt, TimeSeries::DuplicatePolicyEnum::LAST_VALUE ); + endIndex = ts()->getValueIndex( endDt, TimeSeries::DuplicatePolicyEnum::LAST_VALUE ); break; case autogen::TimeIndexPolicy::enum_::EXCLUSIVE: - endIndex = ts() -> getValueIndex( endDt, TimeSeries::DuplicatePolicyEnum::FIRST_VALUE ); - if( endIndex != -1 && ts() -> timeAtIndex( endIndex ) == endDt ) + endIndex = ts()->getValueIndex( endDt, TimeSeries::DuplicatePolicyEnum::FIRST_VALUE ); + if( endIndex != -1 && ts()->timeAtIndex( endIndex ) == endDt ) endIndex += 1; break; default: @@ -233,17 +219,15 @@ int32_t PyInputProxy::computeEndIndex( DateTime endDt, autogen::TimeIndexPolicy return endIndex; } -PyObject *PyInputProxy::valuesAt( ValueType valueType, PyObject *startIndexArg, - PyObject *endIndexArg, - PyObject *startIndexPolicyArg, - PyObject *endIndexPolicyArg ) const +PyObject * PyInputProxy::valuesAt( ValueType valueType, PyObject * startIndexArg, PyObject * endIndexArg, + PyObject * startIndexPolicyArg, PyObject * endIndexPolicyArg ) const { int32_t startIndex, endIndex; - auto startPolicy = static_cast( static_cast( startIndexPolicyArg ) -> enum_ ); - auto endPolicy = static_cast( static_cast( endIndexPolicyArg ) -> enum_ ); + auto startPolicy = static_cast( static_cast( startIndexPolicyArg )->enum_ ); + auto endPolicy = static_cast( static_cast( endIndexPolicyArg )->enum_ ); if( startIndexArg == Py_None ) - startIndex = 1 - ts() -> numTicks(); + startIndex = 1 - ts()->numTicks(); if( endIndexArg == Py_None ) endIndex = 0; @@ -264,7 +248,8 @@ PyObject *PyInputProxy::valuesAt( ValueType valueType, PyObject *startIndexArg, CSP_TRUE_OR_THROW_RUNTIME( endIndex <= 0, "Expected non positive value for values_at end index, got " << endIndex ); CSP_TRUE_OR_THROW_RUNTIME( endIndex >= startIndex, - "Start index (got) " << startIndex << " must come before end index (got) " << endIndex ); + "Start index (got) " << startIndex << " must come before end index (got) " + << endIndex ); // Convert negative value to actual positive index startIndex = -startIndex; endIndex = -endIndex; @@ -275,22 +260,28 @@ PyObject *PyInputProxy::valuesAt( ValueType valueType, PyObject *startIndexArg, if( endPolicy.enum_value() != autogen::TimeIndexPolicy::enum_::INCLUSIVE ) CSP_THROW( InvalidArgument, "Unsupported time index policy for integer indexing: " << endPolicy.name() ); - if( unlikely( startIndex >= ts() -> tickCountPolicy() ) && startIndexArg != Py_None ) - CSP_THROW( RangeError, "buffer index out of range. requesting data at index " << startIndex << " with buffer policy set to " << ts() -> tickCountPolicy() << " ticks in node '" << m_node -> name() << "'" ); - if( unlikely( endIndex >= ts() -> tickCountPolicy() ) ) - CSP_THROW( RangeError, "buffer index out of range. requesting data at index " << endIndex << " with buffer policy set to " << ts() -> tickCountPolicy() << " ticks in node '" << m_node -> name() << "'" ); + if( unlikely( startIndex >= ts()->tickCountPolicy() ) && startIndexArg != Py_None ) + CSP_THROW( RangeError, + "buffer index out of range. requesting data at index " + << startIndex << " with buffer policy set to " << ts()->tickCountPolicy() + << " ticks in node '" << m_node->name() << "'" ); + if( unlikely( endIndex >= ts()->tickCountPolicy() ) ) + CSP_THROW( RangeError, + "buffer index out of range. requesting data at index " + << endIndex << " with buffer policy set to " << ts()->tickCountPolicy() << " ticks in node '" + << m_node->name() << "'" ); startIndex = std::min( static_cast( startIndex ), num_ticks() - 1 ); - endIndex = std::min( static_cast( endIndex ), num_ticks() - 1 ); + endIndex = std::min( static_cast( endIndex ), num_ticks() - 1 ); } else { - CSP_TRUE_OR_THROW_RUNTIME( startIndexArg == Py_None || endIndexArg == Py_None || - Py_TYPE( startIndexArg ) == Py_TYPE( endIndexArg ), + CSP_TRUE_OR_THROW_RUNTIME( startIndexArg == Py_None || endIndexArg == Py_None + || Py_TYPE( startIndexArg ) == Py_TYPE( endIndexArg ), "Start and end index must both be datetime or both be timedelta" ); TimeDelta startTd, endTd; - DateTime startDt, endDt; + DateTime startDt, endDt; if( startIndexArg != Py_None ) { @@ -300,22 +291,27 @@ PyObject *PyInputProxy::valuesAt( ValueType valueType, PyObject *startIndexArg, startTd = startDtd.timedelta(); CSP_TRUE_OR_THROW_RUNTIME( startTd.sign() <= 0, "Positive timedelta is unsupported" ); - if( unlikely( -startTd > ts() -> tickTimeWindowPolicy() ) ) - CSP_THROW( RangeError, "buffer timedelta out of range. requesting data at timedelta " << - PyObjectPtr::incref( startIndexArg ) << " with buffer policy set to " << - PyObjectPtr::own( toPython( ts() -> tickTimeWindowPolicy() ) ) << - " in node '" << m_node -> name() << "'" ); + if( unlikely( -startTd > ts()->tickTimeWindowPolicy() ) ) + CSP_THROW( RangeError, + "buffer timedelta out of range. requesting data at timedelta " + << PyObjectPtr::incref( startIndexArg ) << " with buffer policy set to " + << PyObjectPtr::own( toPython( ts()->tickTimeWindowPolicy() ) ) << " in node '" + << m_node->name() << "'" ); - startIndex = computeStartIndex( m_node -> now() + startTd, startPolicy ); + startIndex = computeStartIndex( m_node->now() + startTd, startPolicy ); } else { startDt = startDtd.datetime(); - CSP_TRUE_OR_THROW_RUNTIME( startDt <= m_node -> now(), "requesting data from future time" ); - if( unlikely( m_node -> now() - startDt > ts() -> tickTimeWindowPolicy() ) ) - CSP_THROW( RangeError, "requested buffer time out of range. requesting datetime " << PyObjectPtr::incref( startIndexArg ) << " at time " << - PyObjectPtr::own( toPython( m_node -> now() ) ) << " with buffer time window policy set to " << - PyObjectPtr::own( toPython( ts() -> tickTimeWindowPolicy() ) )<< " in node '" << m_node -> name() << "'" ); + CSP_TRUE_OR_THROW_RUNTIME( startDt <= m_node->now(), "requesting data from future time" ); + if( unlikely( m_node->now() - startDt > ts()->tickTimeWindowPolicy() ) ) + CSP_THROW( RangeError, + "requested buffer time out of range. requesting datetime " + << PyObjectPtr::incref( startIndexArg ) << " at time " + << PyObjectPtr::own( toPython( m_node->now() ) ) + << " with buffer time window policy set to " + << PyObjectPtr::own( toPython( ts()->tickTimeWindowPolicy() ) ) << " in node '" + << m_node->name() << "'" ); startIndex = computeStartIndex( startDt, startPolicy ); } @@ -331,29 +327,37 @@ PyObject *PyInputProxy::valuesAt( ValueType valueType, PyObject *startIndexArg, if( startIndexArg != Py_None ) CSP_TRUE_OR_THROW_RUNTIME( endTd >= startTd, - "Start timedelta (got " << startTd << ") must come before end timedelta (got " << endTd << ")" ); - - if( unlikely( -endTd > ts() -> tickTimeWindowPolicy() ) ) - CSP_THROW( RangeError, "buffer timedelta out of range. requesting data at timedelta " << - PyObjectPtr::incref( endIndexArg ) << " with buffer policy set to " << - PyObjectPtr::own( toPython( ts() -> tickTimeWindowPolicy() ) ) << - " in node '" << m_node -> name() << "'" ); - - endIndex = computeEndIndex( m_node -> now() + endTd, endPolicy ); + "Start timedelta (got " + << startTd << ") must come before end timedelta (got " << endTd + << ")" ); + + if( unlikely( -endTd > ts()->tickTimeWindowPolicy() ) ) + CSP_THROW( RangeError, + "buffer timedelta out of range. requesting data at timedelta " + << PyObjectPtr::incref( endIndexArg ) << " with buffer policy set to " + << PyObjectPtr::own( toPython( ts()->tickTimeWindowPolicy() ) ) << " in node '" + << m_node->name() << "'" ); + + endIndex = computeEndIndex( m_node->now() + endTd, endPolicy ); } else { endDt = endDtd.datetime(); - CSP_TRUE_OR_THROW_RUNTIME( endDt <= m_node -> now(), "requesting data from future time" ); + CSP_TRUE_OR_THROW_RUNTIME( endDt <= m_node->now(), "requesting data from future time" ); if( startIndexArg != Py_None ) - CSP_TRUE_OR_THROW_RUNTIME( endDt >= startDt, - "Start datetime (got " << startDt << ") must come before end datetime (got " << endDt << ")" ); - - if( unlikely( m_node -> now() - endDt >= ts() -> tickTimeWindowPolicy() ) ) - CSP_THROW( RangeError, "requested buffer time out of range. requesting datetime " << PyObjectPtr::incref( endIndexArg ) << " at time " << - PyObjectPtr::own( toPython( m_node -> now() ) ) << " with buffer time window policy set to " << - PyObjectPtr::own( toPython( ts() -> tickTimeWindowPolicy() ) )<< " in node '" << m_node -> name() << "'" ); + CSP_TRUE_OR_THROW_RUNTIME( + endDt >= startDt, + "Start datetime (got " << startDt << ") must come before end datetime (got " << endDt << ")" ); + + if( unlikely( m_node->now() - endDt >= ts()->tickTimeWindowPolicy() ) ) + CSP_THROW( RangeError, + "requested buffer time out of range. requesting datetime " + << PyObjectPtr::incref( endIndexArg ) << " at time " + << PyObjectPtr::own( toPython( m_node->now() ) ) + << " with buffer time window policy set to " + << PyObjectPtr::own( toPython( ts()->tickTimeWindowPolicy() ) ) << " in node '" + << m_node->name() << "'" ); endIndex = computeEndIndex( endDt, endPolicy ); } @@ -365,9 +369,11 @@ PyObject *PyInputProxy::valuesAt( ValueType valueType, PyObject *startIndexArg, if( startPolicy == autogen::TimeIndexPolicy::EXTRAPOLATE ) { if( !startTd.isNone() ) - return valuesAtIndexToNumpy( valueType, ts(), startIndex, endIndex, startPolicy, endPolicy, m_node -> now() + startTd, m_node -> now() + endTd ); + return valuesAtIndexToNumpy( valueType, ts(), startIndex, endIndex, startPolicy, endPolicy, + m_node->now() + startTd, m_node->now() + endTd ); if( !startDt.isNone() ) - return valuesAtIndexToNumpy( valueType, ts(), startIndex, endIndex, startPolicy, endPolicy, startDt, endDt ); + return valuesAtIndexToNumpy( valueType, ts(), startIndex, endIndex, startPolicy, endPolicy, startDt, + endDt ); } } @@ -378,7 +384,7 @@ static PyObject * PyInputProxy_ticked( PyInputProxy * proxy ) { CSP_BEGIN_METHOD; - return toPython( proxy -> ticked() ); + return toPython( proxy->ticked() ); CSP_RETURN_NONE; } @@ -386,7 +392,7 @@ static PyObject * PyInputProxy_valid( PyInputProxy * proxy ) { CSP_BEGIN_METHOD; - return toPython( proxy -> valid() ); + return toPython( proxy->valid() ); CSP_RETURN_NONE; } @@ -394,13 +400,13 @@ static Py_ssize_t PyInputProxy_count( PyInputProxy * proxy ) { CSP_BEGIN_METHOD; - return proxy -> count(); + return proxy->count(); CSP_RETURN_INT; } static void scheduler_handle_destructor( PyObject * o ) { - Scheduler::Handle * raw = ( Scheduler::Handle * ) PyCapsule_GetPointer( o, "handle" ); + Scheduler::Handle * raw = (Scheduler::Handle *)PyCapsule_GetPointer( o, "handle" ); delete raw; } @@ -413,10 +419,10 @@ static PyObject * PyInputProxy_schedule_alarm( PyInputProxy * proxy, PyObject * if( !PyArg_ParseTuple( args, "OO", &timeOrDelta, &value ) ) return nullptr; - //todo fixme to handle time + // todo fixme to handle time auto dtd = fromPython( timeOrDelta ); - Scheduler::Handle * ret = new Scheduler::Handle( proxy -> scheduleAlarm( dtd, value ) ); + Scheduler::Handle * ret = new Scheduler::Handle( proxy->scheduleAlarm( dtd, value ) ); return PyCapsule_New( ret, "handle", scheduler_handle_destructor ); CSP_RETURN_NONE; @@ -434,10 +440,10 @@ static PyObject * PyInputProxy_reschedule_alarm( PyInputProxy * proxy, PyObject void * raw = PyCapsule_GetPointer( pyHandle, "handle" ); if( !raw ) CSP_THROW( PythonPassthrough, "" ); - auto * handle = ( Scheduler::Handle * ) raw; - auto dtd = fromPython( timeOrDelta ); + auto * handle = (Scheduler::Handle *)raw; + auto dtd = fromPython( timeOrDelta ); - Scheduler::Handle * ret = new Scheduler::Handle( proxy -> rescheduleAlarm( *handle, dtd ) ); + Scheduler::Handle * ret = new Scheduler::Handle( proxy->rescheduleAlarm( *handle, dtd ) ); return PyCapsule_New( ret, "handle", scheduler_handle_destructor ); CSP_RETURN_NONE; } @@ -453,23 +459,23 @@ static PyObject * PyInputProxy_cancel_alarm( PyInputProxy * proxy, PyObject * ar void * raw = PyCapsule_GetPointer( pyHandle, "handle" ); if( !raw ) CSP_THROW( PythonPassthrough, "" ); - auto * handle = ( Scheduler::Handle * ) raw; + auto * handle = (Scheduler::Handle *)raw; - proxy -> cancelAlarm( *handle ); + proxy->cancelAlarm( *handle ); CSP_RETURN_NONE; } static PyObject * PyInputProxy_make_active( PyInputProxy * proxy, PyObject * args ) { CSP_BEGIN_METHOD; - return toPython( proxy -> makeActive() ); + return toPython( proxy->makeActive() ); CSP_RETURN_NONE; } static PyObject * PyInputProxy_make_passive( PyInputProxy * proxy, PyObject * args ) { CSP_BEGIN_METHOD; - return toPython( proxy -> makePassive() ); + return toPython( proxy->makePassive() ); CSP_RETURN_NONE; } @@ -480,16 +486,14 @@ static PyObject * PyInputProxy_set_buffering_policy( PyInputProxy * proxy, PyObj PyObject * tickHistory = nullptr; static const char * kwlist[] = { "tick_count", "tick_history", nullptr }; - if( !PyArg_ParseTupleAndKeywords( args, kwargs, "|O!O", ( char ** ) kwlist, - &PyLong_Type, &tickCount, - &tickHistory ) ) + if( !PyArg_ParseTupleAndKeywords( args, kwargs, "|O!O", (char **)kwlist, &PyLong_Type, &tickCount, &tickHistory ) ) CSP_THROW( PythonPassthrough, "" ); if( !tickCount && !tickHistory ) CSP_THROW( TypeError, "csp.set_buffering_policy expected at least one of tick_count or tick_history" ); - proxy -> setBufferingPolicy( tickCount ? fromPython( tickCount ) : -1, - tickHistory ? fromPython( tickHistory ) : TimeDelta::NONE() ); + proxy->setBufferingPolicy( tickCount ? fromPython( tickCount ) : -1, + tickHistory ? fromPython( tickHistory ) : TimeDelta::NONE() ); CSP_RETURN_NONE; } @@ -500,9 +504,9 @@ static inline PyObject * PyInputProxy_value_at_impl( ValueType valueType, PyInpu PyObject * duplicatePolicyArg; PyObject * defaultValueArg; if( !PyArg_ParseTuple( args, "OOO", &indexArg, &duplicatePolicyArg, &defaultValueArg ) ) - CSP_THROW(RuntimeException, "Invalid arguments parsed to value_at"); + CSP_THROW( RuntimeException, "Invalid arguments parsed to value_at" ); - return proxy->valueAt( valueType, indexArg, duplicatePolicyArg, defaultValueArg); + return proxy->valueAt( valueType, indexArg, duplicatePolicyArg, defaultValueArg ); CSP_RETURN_NONE; } @@ -513,141 +517,141 @@ static inline PyObject * PyInputProxy_values_at_impl( ValueType valueType, PyInp PyObject * endIndexArg; PyObject * startExclusiveArg; PyObject * endExclusiveArg; - if( !PyArg_ParseTuple( args, "OOO!O!", &startIndexArg, &endIndexArg, - &PyCspEnum::PyType, &startExclusiveArg, + if( !PyArg_ParseTuple( args, "OOO!O!", &startIndexArg, &endIndexArg, &PyCspEnum::PyType, &startExclusiveArg, &PyCspEnum::PyType, &endExclusiveArg ) ) CSP_THROW( RuntimeException, "Invalid arguments passed to values_at" ); - return proxy -> valuesAt( valueType, startIndexArg, endIndexArg, startExclusiveArg, endExclusiveArg ); + return proxy->valuesAt( valueType, startIndexArg, endIndexArg, startExclusiveArg, endExclusiveArg ); CSP_RETURN_NONE; } - static PyObject * PyInputProxy_value_at( PyInputProxy * proxy, PyObject * args ) { - return PyInputProxy_value_at_impl(ValueType::VALUE, proxy, args); + return PyInputProxy_value_at_impl( ValueType::VALUE, proxy, args ); } -static PyObject * PyInputProxy_time_at(PyInputProxy *proxy, PyObject *args) +static PyObject * PyInputProxy_time_at( PyInputProxy * proxy, PyObject * args ) { - return PyInputProxy_value_at_impl(ValueType::TIMESTAMP, proxy, args); + return PyInputProxy_value_at_impl( ValueType::TIMESTAMP, proxy, args ); } -static PyObject * PyInputProxy_item_at(PyInputProxy *proxy, PyObject *args) +static PyObject * PyInputProxy_item_at( PyInputProxy * proxy, PyObject * args ) { - return PyInputProxy_value_at_impl(ValueType::TIMESTAMP_VALUE_TUPLE, proxy, args); + return PyInputProxy_value_at_impl( ValueType::TIMESTAMP_VALUE_TUPLE, proxy, args ); } static PyObject * PyInputProxy_values_at( PyInputProxy * proxy, PyObject * args ) { - return PyInputProxy_values_at_impl(ValueType::VALUE, proxy, args); + return PyInputProxy_values_at_impl( ValueType::VALUE, proxy, args ); } -static PyObject * PyInputProxy_times_at(PyInputProxy *proxy, PyObject *args) +static PyObject * PyInputProxy_times_at( PyInputProxy * proxy, PyObject * args ) { - return PyInputProxy_values_at_impl(ValueType::TIMESTAMP, proxy, args); + return PyInputProxy_values_at_impl( ValueType::TIMESTAMP, proxy, args ); } -static PyObject * PyInputProxy_items_at(PyInputProxy *proxy, PyObject *args) +static PyObject * PyInputProxy_items_at( PyInputProxy * proxy, PyObject * args ) { - return PyInputProxy_values_at_impl(ValueType::TIMESTAMP_VALUE_TUPLE, proxy, args); + return PyInputProxy_values_at_impl( ValueType::TIMESTAMP_VALUE_TUPLE, proxy, args ); } -static PyMethodDef PyInputProxy_methods[] = { - {"schedule_alarm", (PyCFunction) PyInputProxy_schedule_alarm, METH_VARARGS, "schedule alarm"}, - {"reschedule_alarm", (PyCFunction) PyInputProxy_reschedule_alarm, METH_VARARGS, "reschedule alarm"}, - {"cancel_alarm", (PyCFunction) PyInputProxy_cancel_alarm, METH_VARARGS, "cancel alarm"}, - {"make_active", (PyCFunction) PyInputProxy_make_active, METH_NOARGS, "make input active"}, - {"make_passive", (PyCFunction) PyInputProxy_make_passive, METH_NOARGS, "make input passive"}, - {"set_buffering_policy", (PyCFunction) PyInputProxy_set_buffering_policy, METH_VARARGS | - METH_KEYWORDS, "set buffering policy"}, - {"value_at", (PyCFunction) PyInputProxy_value_at, METH_VARARGS, "Get historical value from time series"}, - {"time_at", (PyCFunction) PyInputProxy_time_at, METH_VARARGS, "Get historical timestamp from time series"}, - {"item_at", (PyCFunction) PyInputProxy_item_at, METH_VARARGS, "Get historical timestamp and value tuple from time series"}, - {"values_at", (PyCFunction) PyInputProxy_values_at, METH_VARARGS, "Get historical values from time series"}, - {"times_at", (PyCFunction) PyInputProxy_times_at, METH_VARARGS, "Get historical timestamps from time series"}, - {"items_at", (PyCFunction) PyInputProxy_items_at, METH_VARARGS, "Get historical timestamp and value tuples from time series"}, - {NULL} -}; +static PyMethodDef PyInputProxy_methods[] + = { { "schedule_alarm", (PyCFunction)PyInputProxy_schedule_alarm, METH_VARARGS, "schedule alarm" }, + { "reschedule_alarm", (PyCFunction)PyInputProxy_reschedule_alarm, METH_VARARGS, "reschedule alarm" }, + { "cancel_alarm", (PyCFunction)PyInputProxy_cancel_alarm, METH_VARARGS, "cancel alarm" }, + { "make_active", (PyCFunction)PyInputProxy_make_active, METH_NOARGS, "make input active" }, + { "make_passive", (PyCFunction)PyInputProxy_make_passive, METH_NOARGS, "make input passive" }, + { "set_buffering_policy", (PyCFunction)PyInputProxy_set_buffering_policy, METH_VARARGS | METH_KEYWORDS, + "set buffering policy" }, + { "value_at", (PyCFunction)PyInputProxy_value_at, METH_VARARGS, "Get historical value from time series" }, + { "time_at", (PyCFunction)PyInputProxy_time_at, METH_VARARGS, "Get historical timestamp from time series" }, + { "item_at", (PyCFunction)PyInputProxy_item_at, METH_VARARGS, + "Get historical timestamp and value tuple from time series" }, + { "values_at", (PyCFunction)PyInputProxy_values_at, METH_VARARGS, "Get historical values from time series" }, + { "times_at", (PyCFunction)PyInputProxy_times_at, METH_VARARGS, "Get historical timestamps from time series" }, + { "items_at", (PyCFunction)PyInputProxy_items_at, METH_VARARGS, + "Get historical timestamp and value tuples from time series" }, + { NULL } }; static PySequenceMethods PyInputProxy_SeqMethods = { - (lenfunc) PyInputProxy_count, + (lenfunc)PyInputProxy_count, }; static PyNumberMethods PyInputProxy_NumberMethods = { - 0, /* binaryfunc nb_add */ - 0, /* binaryfunc nb_subtract */ - 0, /* binaryfunc nb_multiply */ - 0, /* binaryfunc nb_remainder */ - 0, /* binaryfunc nb_divmod */ - 0, /* ternaryfunc nb_power */ - ( unaryfunc ) PyInputProxy_valid, /* unaryfunc nb_negative */ - ( unaryfunc ) PyInputProxy_ticked, /* unaryfunc nb_positive */ - 0, /* unaryfunc nb_absolute */ - 0, /* inquiry nb_nonzero */ - 0, /* unaryfunc nb_invert */ - 0, /* binaryfunc nb_lshift */ - 0, /* binaryfunc nb_rshift */ - 0, /* binaryfunc nb_and */ - 0, /* binaryfunc nb_xor */ - 0, /* binaryfunc nb_or */ - 0, /* unaryfunc nb_int */ - 0, /* void * reserved */ - 0, /* unaryfunc nb_float */ + 0, /* binaryfunc nb_add */ + 0, /* binaryfunc nb_subtract */ + 0, /* binaryfunc nb_multiply */ + 0, /* binaryfunc nb_remainder */ + 0, /* binaryfunc nb_divmod */ + 0, /* ternaryfunc nb_power */ + (unaryfunc)PyInputProxy_valid, /* unaryfunc nb_negative */ + (unaryfunc)PyInputProxy_ticked, /* unaryfunc nb_positive */ + 0, /* unaryfunc nb_absolute */ + 0, /* inquiry nb_nonzero */ + 0, /* unaryfunc nb_invert */ + 0, /* binaryfunc nb_lshift */ + 0, /* binaryfunc nb_rshift */ + 0, /* binaryfunc nb_and */ + 0, /* binaryfunc nb_xor */ + 0, /* binaryfunc nb_or */ + 0, /* unaryfunc nb_int */ + 0, /* void * reserved */ + 0, /* unaryfunc nb_float */ }; - PyTypeObject PyInputProxy::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyInputProxy", /* tp_name */ - sizeof(PyInputProxy), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - &PyInputProxy_NumberMethods,/* tp_as_number */ - &PyInputProxy_SeqMethods, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp input proxy", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyInputProxy_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ /* Low-level free-memory routine */ - 0, /* tp_is_gc */ /* For PyObject_IS_GC */ - 0, /* tp_bases */ - 0, /* tp_mro */ /* method resolution order */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0 /* tp_finalize */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyInputProxy", /* tp_name */ + sizeof( PyInputProxy ), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + &PyInputProxy_NumberMethods, /* tp_as_number */ + &PyInputProxy_SeqMethods, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp input proxy", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyInputProxy_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + 0, + /* tp_free */ /* Low-level free-memory routine */ + 0, + /* tp_is_gc */ /* For PyObject_IS_GC */ + 0, /* tp_bases */ + 0, + /* tp_mro */ /* method resolution order */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0 /* tp_finalize */ }; REGISTER_TYPE_INIT( &PyInputProxy::PyType, "PyInputProxy" ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyManagedSimInputAdapter.cpp b/cpp/csp/python/PyManagedSimInputAdapter.cpp index 575851be..56849d2c 100644 --- a/cpp/csp/python/PyManagedSimInputAdapter.cpp +++ b/cpp/csp/python/PyManagedSimInputAdapter.cpp @@ -2,10 +2,10 @@ #include #include #include -#include #include #include #include +#include #include namespace csp::python @@ -14,17 +14,17 @@ namespace csp::python class PyManagedSimInputAdapter : public ManagedSimInputAdapter { public: - PyManagedSimInputAdapter( Engine * engine, AdapterManager * manager, - PyObjectPtr pyadapter, PyObject * pyType, - PushMode pushMode ) : ManagedSimInputAdapter( engine, pyTypeAsCspType( pyType ), manager, pushMode ), - m_pyadapter( pyadapter ), - m_pyType( PyObjectPtr::incref( ( PyObject * ) pyType ) ) + PyManagedSimInputAdapter( Engine * engine, AdapterManager * manager, PyObjectPtr pyadapter, PyObject * pyType, + PushMode pushMode ) + : ManagedSimInputAdapter( engine, pyTypeAsCspType( pyType ), manager, pushMode ) + , m_pyadapter( pyadapter ) + , m_pyType( PyObjectPtr::incref( (PyObject *)pyType ) ) { } void start( DateTime start, DateTime end ) override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "start", "OO", + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "start", "OO", PyObjectPtr::own( toPython( start ) ).ptr(), PyObjectPtr::own( toPython( end ) ).ptr() ) ); if( !rv.ptr() ) @@ -33,7 +33,7 @@ class PyManagedSimInputAdapter : public ManagedSimInputAdapter void stop() override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "stop", nullptr ) ); + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "stop", nullptr ) ); if( !rv.ptr() ) CSP_THROW( PythonPassthrough, "" ); @@ -50,129 +50,132 @@ template class TypedPyManagedSimInputAdapter : public PyManagedSimInputAdapter { public: - TypedPyManagedSimInputAdapter( Engine * engine, AdapterManager * manager, PyObjectPtr pyadapter, PyObject * pyType, PushMode pushMode ): - PyManagedSimInputAdapter( engine, manager, pyadapter, pyType, pushMode ) + TypedPyManagedSimInputAdapter( Engine * engine, AdapterManager * manager, PyObjectPtr pyadapter, PyObject * pyType, + PushMode pushMode ) + : PyManagedSimInputAdapter( engine, manager, pyadapter, pyType, pushMode ) { - } - + void pushPyTick( PyObject * value ) override { try { - if( !validatePyType( this -> dataType(), m_pyType.ptr(), value ) ) + if( !validatePyType( this->dataType(), m_pyType.ptr(), value ) ) CSP_THROW( TypeError, "" ); - + pushTick( fromPython( value, *dataType() ) ); } catch( const TypeError & ) { - CSP_THROW( TypeError, "\"" << Py_TYPE( m_pyadapter.ptr() ) -> tp_name << "\" managed sim adapter expected output type to be of type \"" - << pyTypeToString( m_pyType.ptr() ) << "\" got type \"" << Py_TYPE( value ) -> tp_name << "\"" ); + CSP_THROW( TypeError, + "\"" << Py_TYPE( m_pyadapter.ptr() )->tp_name + << "\" managed sim adapter expected output type to be of type \"" + << pyTypeToString( m_pyType.ptr() ) << "\" got type \"" << Py_TYPE( value )->tp_name + << "\"" ); } - } }; struct PyManagedSimInputAdapter_PyObject { - PyObject_HEAD - PyManagedSimInputAdapter * adapter; + PyObject_HEAD PyManagedSimInputAdapter * adapter; static PyObject * pushTick( PyManagedSimInputAdapter_PyObject * self, PyObject * pyValue ) { CSP_BEGIN_METHOD; - self -> adapter -> pushPyTick( pyValue ); + self->adapter->pushPyTick( pyValue ); CSP_RETURN_NONE; } static PyTypeObject PyType; }; - -static PyMethodDef PyManagedSimInputAdapter_PyObject_methods[] = { - { "push_tick", (PyCFunction) PyManagedSimInputAdapter_PyObject::pushTick, METH_O, "push new tick" }, - {NULL} -}; + +static PyMethodDef PyManagedSimInputAdapter_PyObject_methods[] + = { { "push_tick", (PyCFunction)PyManagedSimInputAdapter_PyObject::pushTick, METH_O, "push new tick" }, { NULL } }; PyTypeObject PyManagedSimInputAdapter_PyObject::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.Pymanagedsiminputadapter", /* tp_name */ - sizeof(PyManagedSimInputAdapter_PyObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - "csp maanged sim adapter", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyManagedSimInputAdapter_PyObject_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.Pymanagedsiminputadapter", /* tp_name */ + sizeof( PyManagedSimInputAdapter_PyObject ), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + "csp maanged sim adapter", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyManagedSimInputAdapter_PyObject_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ 0, PyType_GenericNew, - 0, /* tp_free */ /* Low-level free-memory routine */ - 0, /* tp_is_gc */ /* For PyObject_IS_GC */ - 0, /* tp_bases */ - 0, /* tp_mro */ /* method resolution order */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0 /* tp_finalize */ + 0, + /* tp_free */ /* Low-level free-memory routine */ + 0, + /* tp_is_gc */ /* For PyObject_IS_GC */ + 0, /* tp_bases */ + 0, + /* tp_mro */ /* method resolution order */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0 /* tp_finalize */ }; REGISTER_TYPE_INIT( &PyManagedSimInputAdapter_PyObject::PyType, "PyManagedSimInputAdapter" ); -static InputAdapter * pymanagedsimadapter_creator( csp::AdapterManager * manager, PyEngine * engine, - PyObject * pyType, PushMode pushMode, PyObject * args ) +static InputAdapter * pymanagedsimadapter_creator( csp::AdapterManager * manager, PyEngine * engine, PyObject * pyType, + PushMode pushMode, PyObject * args ) { PyTypeObject * pyAdapterType = nullptr; - PyObject * adapterArgs = nullptr; - if( !PyArg_ParseTuple( args, "O!O!", &PyType_Type, &pyAdapterType, - &PyTuple_Type, &adapterArgs ) ) + PyObject * adapterArgs = nullptr; + if( !PyArg_ParseTuple( args, "O!O!", &PyType_Type, &pyAdapterType, &PyTuple_Type, &adapterArgs ) ) CSP_THROW( PythonPassthrough, "" ); if( !PyType_IsSubtype( pyAdapterType, &PyManagedSimInputAdapter_PyObject::PyType ) ) - CSP_THROW( TypeError, "Expected PyManagedSimInputAdapter derived type, got " << pyAdapterType -> tp_name ); + CSP_THROW( TypeError, "Expected PyManagedSimInputAdapter derived type, got " << pyAdapterType->tp_name ); - PyManagedSimInputAdapter_PyObject * pyAdapter = ( PyManagedSimInputAdapter_PyObject * ) PyObject_Call( ( PyObject * ) pyAdapterType, adapterArgs, nullptr ); + PyManagedSimInputAdapter_PyObject * pyAdapter + = (PyManagedSimInputAdapter_PyObject *)PyObject_Call( (PyObject *)pyAdapterType, adapterArgs, nullptr ); if( !pyAdapter ) CSP_THROW( PythonPassthrough, "" ); - switchPyType( pyType, - [&]( auto tag ) - { - pyAdapter -> adapter = engine -> engine() -> createOwnedObject>( - manager, PyObjectPtr::own( ( PyObject * ) pyAdapter ), pyType, pushMode ); - } ); + switchPyType( + pyType, + [&]( auto tag ) + { + pyAdapter->adapter + = engine->engine()->createOwnedObject>( + manager, PyObjectPtr::own( (PyObject *)pyAdapter ), pyType, pushMode ); + } ); - return pyAdapter -> adapter; + return pyAdapter->adapter; } REGISTER_INPUT_ADAPTER( _managedsimadapter, pymanagedsimadapter_creator ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyNode.cpp b/cpp/csp/python/PyNode.cpp index d13e1b80..3dac5128 100644 --- a/cpp/csp/python/PyNode.cpp +++ b/cpp/csp/python/PyNode.cpp @@ -3,17 +3,17 @@ #include #include #include -#include #include #include -#include -#include #include +#include +#include #include #include +#include -#include #include +#include #if !IS_PRE_PYTHON_3_11 #include @@ -23,18 +23,18 @@ namespace csp::python { -static const std::string NODEREF_VAR="node_p"; -static const std::string INPUT_VAR_VAR="input_var"; -static const std::string INPUT_PROXY_VAR="input_proxy"; -static const std::string OUTPUT_PROXY_VAR="output_proxy"; +static const std::string NODEREF_VAR = "node_p"; +static const std::string INPUT_VAR_VAR = "input_var"; +static const std::string INPUT_PROXY_VAR = "input_proxy"; +static const std::string OUTPUT_PROXY_VAR = "output_proxy"; -static const uint32_t ACTIVE_PCOUNT=std::numeric_limits::max(); +static const uint32_t ACTIVE_PCOUNT = std::numeric_limits::max(); -PyNode::PyNode( csp::Engine * engine, PyObjectPtr gen, PyObjectPtr inputs, PyObjectPtr outputs, - NodeDef def ) : csp::Node( def, engine ), - m_gen( gen ), - m_localVars( nullptr ), - m_passiveCounts( nullptr ) +PyNode::PyNode( csp::Engine * engine, PyObjectPtr gen, PyObjectPtr inputs, PyObjectPtr outputs, NodeDef def ) + : csp::Node( def, engine ) + , m_gen( gen ) + , m_localVars( nullptr ) + , m_passiveCounts( nullptr ) { init( inputs, outputs ); } @@ -47,69 +47,69 @@ PyNode::~PyNode() void PyNode::init( PyObjectPtr inputs, PyObjectPtr outputs ) { - PyGenObject * pygen = ( PyGenObject * ) m_gen.ptr(); - - //call gen first yield to setup locals + PyGenObject * pygen = (PyGenObject *)m_gen.ptr(); + + // call gen first yield to setup locals call_gen(); - //python stack locals are laid out as - //node proxy - //in proxies - //out proxies - //in locals ( non-basket only ) + // python stack locals are laid out as + // node proxy + // in proxies + // out proxies + // in locals ( non-basket only ) - m_localVars = ( PyObject *** ) calloc( numInputs(), sizeof( PyObject ** ) ); + m_localVars = (PyObject ***)calloc( numInputs(), sizeof( PyObject ** ) ); - //printf( "Starting %s slots: %ld rank: %d\n", name(), slots, rank() ); + // printf( "Starting %s slots: %ld rank: %d\n", name(), slots, rank() ); #if IS_PRE_PYTHON_3_11 - PyCodeObject * code = ( PyCodeObject * ) pygen -> gi_code; - Py_ssize_t numCells = PyTuple_GET_SIZE( code -> co_cellvars ); - size_t cell2argIdx = 0; - for( int stackloc = code -> co_argcount; stackloc < code -> co_nlocals + numCells; ++stackloc ) + PyCodeObject * code = (PyCodeObject *)pygen->gi_code; + Py_ssize_t numCells = PyTuple_GET_SIZE( code->co_cellvars ); + size_t cell2argIdx = 0; + for( int stackloc = code->co_argcount; stackloc < code->co_nlocals + numCells; ++stackloc ) { - PyObject **var = &pygen -> gi_frame -> f_localsplus[stackloc]; + PyObject ** var = &pygen->gi_frame->f_localsplus[stackloc]; bool isCell = *var && PyCell_Check( *var ); - //printf( "RBA: stack: %d var: ", stackloc ); - //PyObject_Print( *var, stdout, 0 ); - //printf( " isCell: %d\n", isCell ); + // printf( "RBA: stack: %d var: ", stackloc ); + // PyObject_Print( *var, stdout, 0 ); + // printf( " isCell: %d\n", isCell ); if( isCell ) { - //might be a scalar argument cell - if( code -> co_cell2arg && - code -> co_cell2arg[ cell2argIdx++ ] != CO_CELL_NOT_AN_ARG ) + // might be a scalar argument cell + if( code->co_cell2arg && code->co_cell2arg[cell2argIdx++] != CO_CELL_NOT_AN_ARG ) continue; - var = &( ( ( PyCellObject * ) *var ) -> ob_ref ); + var = &( ( (PyCellObject *)*var )->ob_ref ); } -//PY311+ changes +// PY311+ changes #else - _PyInterpreterFrame * frame = ( _PyInterpreterFrame * ) pygen -> gi_iframe; - PyCodeObject * code = frame -> f_code; - int localPlusIndex = 0; - for( int stackloc = code -> co_argcount; stackloc < code -> co_nlocalsplus; ++stackloc, ++localPlusIndex ) + _PyInterpreterFrame * frame = (_PyInterpreterFrame *)pygen->gi_iframe; + PyCodeObject * code = frame->f_code; + int localPlusIndex = 0; + for( int stackloc = code->co_argcount; stackloc < code->co_nlocalsplus; ++stackloc, ++localPlusIndex ) { - PyObject **var = &frame -> localsplus[stackloc]; + PyObject ** var = &frame->localsplus[stackloc]; - auto kind = _PyLocals_GetKind(code -> co_localspluskinds, localPlusIndex ); - bool isCell = *var && PyCell_Check(*var); + auto kind = _PyLocals_GetKind( code->co_localspluskinds, localPlusIndex ); + bool isCell = *var && PyCell_Check( *var ); - //printf( "RBA: stack: %d idx: %d var: ", stackloc, localPlusIndex ); - //PyObject_Print( *var, stdout, 0 ); - //printf( " isCell: %d kind: %x\n", isCell, kind ); + // printf( "RBA: stack: %d idx: %d var: ", stackloc, localPlusIndex ); + // PyObject_Print( *var, stdout, 0 ); + // printf( " isCell: %d kind: %x\n", isCell, kind ); if( isCell ) - var = &( ( ( PyCellObject * ) *var ) -> ob_ref ); + var = &( ( (PyCellObject *)*var )->ob_ref ); else if( kind == CO_FAST_CELL ) continue; -#endif - //null var indicates a stack slot for a local state variable ( state hasnt initialized yet ) - //we can skip those +#endif + // null var indicates a stack slot for a local state variable ( state hasnt initialized yet ) + // we can skip those if( !*var ) continue; if( !PyTuple_Check( *var ) ) - CSP_THROW( TypeError, "expected tuple types in stack, got " << (*var) -> ob_type -> tp_name << " in node " << name() ); + CSP_THROW( TypeError, + "expected tuple types in stack, got " << ( *var )->ob_type->tp_name << " in node " << name() ); std::string vartype = PyUnicode_AsUTF8( PyTuple_GET_ITEM( *var, 0 ) ); int index = fromPython( PyTuple_GET_ITEM( *var, 1 ) ); @@ -118,28 +118,28 @@ void PyNode::init( PyObjectPtr inputs, PyObjectPtr outputs ) { CSP_ASSERT( !isInputBasket( index ) ); - m_localVars[ index ] = var; - //These vars will be "deleted" from the python stack after start + m_localVars[index] = var; + // These vars will be "deleted" from the python stack after start continue; } - //decref tuple at this point its no longer needed and will be replaced + // decref tuple at this point its no longer needed and will be replaced Py_DECREF( *var ); PyObject * newvalue = nullptr; if( vartype == NODEREF_VAR ) - newvalue = toPython( reinterpret_cast( static_cast(this) ) ); + newvalue = toPython( reinterpret_cast( static_cast( this ) ) ); else if( vartype == INPUT_PROXY_VAR ) { PyObject * inputType = PyTuple_GET_ITEM( inputs.ptr(), index ); - //if basket + // if basket if( PyTuple_Check( inputType ) ) { PyObject * shape = PyTuple_GET_ITEM( inputType, 0 ); if( shape == Py_None ) { - //None shape is used to flag dynamic baskets + // None shape is used to flag dynamic baskets initInputBasket( index, 0, true ); newvalue = PyDynamicBasketInputProxy::create( this, index ); } @@ -152,7 +152,9 @@ void PyNode::init( PyObjectPtr inputs, PyObjectPtr outputs ) else { if( !PyList_Check( shape ) ) - CSP_THROW( TypeError, "Expected input basket shape as int or list, got " << Py_TYPE( shape ) -> tp_name << " on node \"" << name() << "\"" ); + CSP_THROW( TypeError, + "Expected input basket shape as int or list, got " + << Py_TYPE( shape )->tp_name << " on node \"" << name() << "\"" ); size_t basketSize = PyList_GET_SIZE( shape ); initInputBasket( index, basketSize, false ); newvalue = PyDictBasketInputProxy::create( this, index, shape ); @@ -167,28 +169,32 @@ void PyNode::init( PyObjectPtr inputs, PyObjectPtr outputs ) { PyObject * outputType = PyTuple_GET_ITEM( outputs.ptr(), index ); - //if basket type + // if basket type if( PyTuple_Check( outputType ) ) { - PyObject * shape = (PyObject * ) PyTuple_GET_ITEM( outputType, 0 ); - outputType = PyTuple_GET_ITEM( outputType, 1 ); + PyObject * shape = (PyObject *)PyTuple_GET_ITEM( outputType, 0 ); + outputType = PyTuple_GET_ITEM( outputType, 1 ); if( shape == Py_None ) { - //None shape is used to flag dynamic baskets + // None shape is used to flag dynamic baskets newvalue = PyDynamicBasketOutputProxy::create( outputType, this, index ); } else if( PyLong_Check( shape ) ) { - std::uint64_t basketSize = fromPython( shape ); + std::uint64_t basketSize = fromPython( shape ); if( basketSize > OutputId::maxBasketElements() ) - CSP_THROW( ValueError, "output basket size exceeds limit of " << OutputId::maxBasketElements() << " on node \"" << name() << "\""); + CSP_THROW( ValueError, + "output basket size exceeds limit of " << OutputId::maxBasketElements() + << " on node \"" << name() << "\"" ); newvalue = PyListBasketOutputProxy::create( outputType, this, index, basketSize ); } else { if( !PyList_Check( shape ) ) - CSP_THROW( TypeError, "Expected output basket shape as int or list, got " << Py_TYPE( shape ) -> tp_name << " on node \"" << name() << "\"" ); + CSP_THROW( TypeError, + "Expected output basket shape as int or list, got " + << Py_TYPE( shape )->tp_name << " on node \"" << name() << "\"" ); newvalue = PyDictBasketOutputProxy::create( outputType, this, index, shape ); } } @@ -206,16 +212,16 @@ void PyNode::init( PyObjectPtr inputs, PyObjectPtr outputs ) void PyNode::start() { - //yield to execute start block, if any + // yield to execute start block, if any call_gen(); } void PyNode::stop() { - if( this -> rootEngine() -> interrupted() && PyErr_CheckSignals() == -1 ) + if( this->rootEngine()->interrupted() && PyErr_CheckSignals() == -1 ) { // When an interrupt occurs a KeyboardInterrupt exception is raised in Python, which we need to clear - // before calling "close" on the generator. Else, the close method will fail due to the unhandled + // before calling "close" on the generator. Else, the close method will fail due to the unhandled // exception, and we lose the state of the generator before the "finally" block that calls stop() is executed. PyErr_Clear(); } @@ -227,26 +233,25 @@ void PyNode::stop() PyNode * PyNode::create( PyEngine * pyengine, PyObject * inputs, PyObject * outputs, PyObject * gen ) { - //parse inputs/outputs, create outputs time series, etc - Py_ssize_t numInputs = PyTuple_GET_SIZE( inputs ); + // parse inputs/outputs, create outputs time series, etc + Py_ssize_t numInputs = PyTuple_GET_SIZE( inputs ); Py_ssize_t numOutputs = PyTuple_GET_SIZE( outputs ); - + if( size_t( numInputs ) >= InputId::maxId() ) CSP_THROW( ValueError, "number of inputs exceeds limit of " << InputId::maxBasketElements() ); if( size_t( numOutputs ) > OutputId::maxId() ) CSP_THROW( ValueError, "number of outputs exceeds limit of " << OutputId::maxBasketElements() ); - return pyengine -> engine() -> createOwnedObject( PyObjectPtr::incref( gen ), - PyObjectPtr::incref( inputs ), - PyObjectPtr::incref( outputs ), - NodeDef( numInputs, numOutputs ) ); + return pyengine->engine()->createOwnedObject( PyObjectPtr::incref( gen ), PyObjectPtr::incref( inputs ), + PyObjectPtr::incref( outputs ), + NodeDef( numInputs, numOutputs ) ); } inline bool PyNode::makeActive( InputId id ) { if( likely( m_passiveCounts != nullptr ) ) - m_passiveCounts[ id.id ] = ACTIVE_PCOUNT; + m_passiveCounts[id.id] = ACTIVE_PCOUNT; return Node::makeActive( id ); } @@ -256,30 +261,27 @@ inline bool PyNode::makePassive( InputId id ) if( unlikely( m_passiveCounts == nullptr ) ) { // Allocate the passive conversion count array upon first csp.make_passive call - m_passiveCounts = ( uint32_t* ) malloc( sizeof( uint32_t ) * numInputs() ); + m_passiveCounts = (uint32_t *)malloc( sizeof( uint32_t ) * numInputs() ); for( size_t i = 0; i < numInputs(); i++ ) - m_passiveCounts[ i ] = ACTIVE_PCOUNT; + m_passiveCounts[i] = ACTIVE_PCOUNT; } - m_passiveCounts[ id.id ] = input( id ) -> count(); + m_passiveCounts[id.id] = input( id )->count(); return Node::makePassive( id ); } void PyNode::createAlarm( CspTypePtr & type, size_t id ) { - //For alarms in PyNode we intentionally create them as PyObjectPtr. We know they arent exposed - //to other nodes so no point in converting back and forth to/from python - this -> Node::createAlarm( CspType::DIALECT_GENERIC(), id ); + // For alarms in PyNode we intentionally create them as PyObjectPtr. We know they arent exposed + // to other nodes so no point in converting back and forth to/from python + this->Node::createAlarm( CspType::DIALECT_GENERIC(), id ); } -const char * PyNode::name() const -{ - return PyUnicode_AsUTF8( ( ( PyGenObject * ) m_gen.ptr() ) -> gi_name ); -} +const char * PyNode::name() const { return PyUnicode_AsUTF8( ( (PyGenObject *)m_gen.ptr() )->gi_name ); } void PyNode::call_gen() { - if( !m_gen -> ob_type -> tp_iternext( m_gen.ptr() ) ) + if( !m_gen->ob_type->tp_iternext( m_gen.ptr() ) ) CSP_THROW( PythonPassthrough, "" ); } @@ -290,17 +292,18 @@ void PyNode::executeImpl() if( !isInputBasket( idx ) ) { // we only need to update locals for inputs that have changed since the last node call - // thus: we convert active inputs that ticked this cycle or passive inputs that ticked since we last converted them - auto * ts = tsinput( idx ); - auto count = ts -> count(); - bool passiveConvert = ( m_passiveCounts != nullptr && m_passiveCounts[ idx ] < count ); + // thus: we convert active inputs that ticked this cycle or passive inputs that ticked since we last + // converted them + auto * ts = tsinput( idx ); + auto count = ts->count(); + bool passiveConvert = ( m_passiveCounts != nullptr && m_passiveCounts[idx] < count ); if( tsinputTicked( idx ) || passiveConvert ) { - Py_XDECREF( *m_localVars[ idx ] ); - *m_localVars[ idx ] = lastValueToPython( ts ); + Py_XDECREF( *m_localVars[idx] ); + *m_localVars[idx] = lastValueToPython( ts ); if( passiveConvert ) - m_passiveCounts[ idx ] = count; + m_passiveCounts[idx] = count; } } } @@ -317,11 +320,8 @@ static PyObject * PyNode_create( PyObject * module, PyObject * args ) PyObject * outputs; PyObject * gen; - if( !PyArg_ParseTuple( args, "O!O!O!O!", - &PyEngine::PyType, &engine, - &PyTuple_Type, &inputs, - &PyTuple_Type, &outputs, - &PyGen_Type, &gen ) ) + if( !PyArg_ParseTuple( args, "O!O!O!O!", &PyEngine::PyType, &engine, &PyTuple_Type, &inputs, &PyTuple_Type, + &outputs, &PyGen_Type, &gen ) ) CSP_THROW( PythonPassthrough, "" ); auto node = PyNode::create( engine, inputs, outputs, gen ); @@ -331,4 +331,4 @@ static PyObject * PyNode_create( PyObject * module, PyObject * args ) REGISTER_MODULE_METHOD( "PyNode", PyNode_create, METH_VARARGS, "PyNode" ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyNodeWrapper.cpp b/cpp/csp/python/PyNodeWrapper.cpp index df759072..a70943fb 100644 --- a/cpp/csp/python/PyNodeWrapper.cpp +++ b/cpp/csp/python/PyNodeWrapper.cpp @@ -1,180 +1,179 @@ #include #include -#include #include +#include #include #include namespace csp::python { -PyNodeWrapper *PyNodeWrapper::create(csp::Node *node) +PyNodeWrapper * PyNodeWrapper::create( csp::Node * node ) { - PyNodeWrapper *wrapper = (PyNodeWrapper *) PyType.tp_new(&PyType, nullptr, nullptr); - new(wrapper) PyNodeWrapper(node); + PyNodeWrapper * wrapper = (PyNodeWrapper *)PyType.tp_new( &PyType, nullptr, nullptr ); + new( wrapper ) PyNodeWrapper( node ); return wrapper; } - -static PyObject *PyNodeWrapper_linkFrom(PyNodeWrapper *self, PyObject *args) +static PyObject * PyNodeWrapper_linkFrom( PyNodeWrapper * self, PyObject * args ) { - CSP_BEGIN_METHOD ; + CSP_BEGIN_METHOD; - //output_idx, output_basket_idx, node_to, input_idx, input_basket_idx + // output_idx, output_basket_idx, node_to, input_idx, input_basket_idx int outputIdx, outputBasketIdx, inputIdx, inputBasketIdx; - PyObject *source; - if (!PyArg_ParseTuple(args, "Oiiii", &source, - &outputIdx, &outputBasketIdx, - &inputIdx, &inputBasketIdx)) + PyObject * source; + if( !PyArg_ParseTuple( args, "Oiiii", &source, &outputIdx, &outputBasketIdx, &inputIdx, &inputBasketIdx ) ) return nullptr; - OutputId outputId(outputIdx, outputBasketIdx); - InputId inputId(inputIdx, inputBasketIdx); + OutputId outputId( outputIdx, outputBasketIdx ); + InputId inputId( inputIdx, inputBasketIdx ); - if( PyType_IsSubtype(Py_TYPE(source), &PyNodeWrapper::PyType) ) + if( PyType_IsSubtype( Py_TYPE( source ), &PyNodeWrapper::PyType ) ) { - auto * sourceNode = static_cast( source ) -> node(); - if( self -> node() -> isInputBasket( inputIdx ) && self -> node() -> inputBasket( inputIdx ) -> isDynamicBasket() ) + auto * sourceNode = static_cast( source )->node(); + if( self->node()->isInputBasket( inputIdx ) && self->node()->inputBasket( inputIdx )->isDynamicBasket() ) { - CSP_ASSERT( sourceNode -> outputBasket( outputIdx ) -> isDynamicBasket() ); + CSP_ASSERT( sourceNode->outputBasket( outputIdx )->isDynamicBasket() ); - auto * outputBasket = sourceNode -> outputBasket( outputIdx ); - static_cast( outputBasket ) -> linkInputBasket( self -> node(), inputIdx ); + auto * outputBasket = sourceNode->outputBasket( outputIdx ); + static_cast( outputBasket )->linkInputBasket( self->node(), inputIdx ); } else - self -> node() -> link( sourceNode -> output( outputId ), inputId ); - } - else if( PyType_IsSubtype(Py_TYPE(source), &PyInputAdapterWrapper::PyType) ) + self->node()->link( sourceNode->output( outputId ), inputId ); + } + else if( PyType_IsSubtype( Py_TYPE( source ), &PyInputAdapterWrapper::PyType ) ) { - self -> node() -> link( static_cast( source ) -> adapter(), inputId ); - } + self->node()->link( static_cast( source )->adapter(), inputId ); + } else - CSP_THROW(TypeError, "link_from expected PyNode or PyInputAdapter as source, got " << Py_TYPE( source ) -> tp_name); + CSP_THROW( TypeError, + "link_from expected PyNode or PyInputAdapter as source, got " << Py_TYPE( source )->tp_name ); CSP_RETURN_NONE; } -static PyObject *PyNodeWrapper_createAlarm(PyNodeWrapper *self, PyObject *args) +static PyObject * PyNodeWrapper_createAlarm( PyNodeWrapper * self, PyObject * args ) { - CSP_BEGIN_METHOD ; + CSP_BEGIN_METHOD; int inputIdx; - PyObject *type; - if (!PyArg_ParseTuple(args, "iO", &inputIdx, &type)) + PyObject * type; + if( !PyArg_ParseTuple( args, "iO", &inputIdx, &type ) ) return nullptr; auto & cspType = pyTypeAsCspType( type ); - self -> node() -> createAlarm( cspType, inputIdx ); + self->node()->createAlarm( cspType, inputIdx ); CSP_RETURN_NONE; } -static PyObject *PyNodeWrapper_createOutput(PyNodeWrapper *self, PyObject *args) +static PyObject * PyNodeWrapper_createOutput( PyNodeWrapper * self, PyObject * args ) { - CSP_BEGIN_METHOD ; + CSP_BEGIN_METHOD; int index; - PyObject *type; + PyObject * type; if( !PyArg_ParseTuple( args, "iO", &index, &type ) ) return nullptr; if( PyTuple_Check( type ) ) { - PyObject * shape = (PyObject * ) PyTuple_GET_ITEM( type, 0 ); - type = PyTuple_GET_ITEM( type, 1 ); - auto & cspType = pyTypeAsCspType( type ); + PyObject * shape = (PyObject *)PyTuple_GET_ITEM( type, 0 ); + type = PyTuple_GET_ITEM( type, 1 ); + auto & cspType = pyTypeAsCspType( type ); if( shape == Py_None ) { - self -> node() -> createDynamicBasketOutput( cspType, index ); + self->node()->createDynamicBasketOutput( cspType, index ); } else { - //static Basket outputs + // static Basket outputs std::uint64_t basketSize; if( PyLong_Check( shape ) ) basketSize = fromPython( shape ); else { if( !PyList_Check( shape ) ) - CSP_THROW( TypeError, "Expected basket shape as int or list, got " << Py_TYPE( shape ) -> tp_name ); + CSP_THROW( TypeError, "Expected basket shape as int or list, got " << Py_TYPE( shape )->tp_name ); basketSize = PyList_GET_SIZE( shape ); } - self -> node() -> createBasketOutput( cspType, index, basketSize ); + self->node()->createBasketOutput( cspType, index, basketSize ); } } else { auto & cspType = pyTypeAsCspType( type ); - self -> node() -> createOutput( cspType, index ); + self->node()->createOutput( cspType, index ); } - + CSP_RETURN_NONE; } -static PyObject *PyNodeWrapper_tp_str(PyNodeWrapper *self) +static PyObject * PyNodeWrapper_tp_str( PyNodeWrapper * self ) { CSP_BEGIN_METHOD - return PyUnicode_FromString( self -> node() -> name() ); + return PyUnicode_FromString( self->node()->name() ); CSP_RETURN_NULL } -static PyMethodDef PyNodeWrapper_methods[] = { - {"link_from", (PyCFunction) PyNodeWrapper_linkFrom, METH_VARARGS, "links node's output to target's input"}, - {"create_alarm", (PyCFunction) PyNodeWrapper_createAlarm, METH_VARARGS, "create a single alarm input"}, - {"create_output", (PyCFunction) PyNodeWrapper_createOutput, METH_VARARGS, "create an output"}, - {NULL} -}; +static PyMethodDef PyNodeWrapper_methods[] + = { { "link_from", (PyCFunction)PyNodeWrapper_linkFrom, METH_VARARGS, "links node's output to target's input" }, + { "create_alarm", (PyCFunction)PyNodeWrapper_createAlarm, METH_VARARGS, "create a single alarm input" }, + { "create_output", (PyCFunction)PyNodeWrapper_createOutput, METH_VARARGS, "create an output" }, + { NULL } }; PyTypeObject PyNodeWrapper::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyNodeWrapper", /* tp_name */ - sizeof(PyNodeWrapper), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - (reprfunc) PyNodeWrapper_tp_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp node wrapper", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyNodeWrapper_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ /* Low-level free-memory routine */ - 0, /* tp_is_gc */ /* For PyObject_IS_GC */ - 0, /* tp_bases */ - 0, /* tp_mro */ /* method resolution order */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0 /* tp_finalize */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyNodeWrapper", /* tp_name */ + sizeof( PyNodeWrapper ), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + (reprfunc)PyNodeWrapper_tp_str, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp node wrapper", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyNodeWrapper_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + 0, + /* tp_free */ /* Low-level free-memory routine */ + 0, + /* tp_is_gc */ /* For PyObject_IS_GC */ + 0, /* tp_bases */ + 0, + /* tp_mro */ /* method resolution order */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0 /* tp_finalize */ }; -REGISTER_TYPE_INIT(&PyNodeWrapper::PyType, "PyNodeWrapper"); +REGISTER_TYPE_INIT( &PyNodeWrapper::PyType, "PyNodeWrapper" ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyNumbaNode.cpp b/cpp/csp/python/PyNumbaNode.cpp index 554a61f2..8e537b60 100644 --- a/cpp/csp/python/PyNumbaNode.cpp +++ b/cpp/csp/python/PyNumbaNode.cpp @@ -3,105 +3,101 @@ #include #include #include +#include +#include #include #include -#include #include -#include -#include #include #include #include +#include #include - #ifdef DEBUG_NUMBA_NODE #include -#define DEBUG_ONLY(v) do {v;} while(false) +#define DEBUG_ONLY( v ) \ + do \ + { \ + v; \ + } while( false ) #else -#define DEBUG_ONLY(v) do {} while(false) +#define DEBUG_ONLY( v ) \ + do \ + { \ + } while( false ) #endif -#define DEBUG_PRINT(v) DEBUG_ONLY(std::cout << v << std::endl) -#define DEBUG_PRINT_WITH_LOC(v) DEBUG_PRINT(__FILE__ << ':' << __LINE__ << ": " << v) - +#define DEBUG_PRINT( v ) DEBUG_ONLY( std::cout << v << std::endl ) +#define DEBUG_PRINT_WITH_LOC( v ) DEBUG_PRINT( __FILE__ << ':' << __LINE__ << ": " << v ) namespace csp::python { -PyNumbaNode::PyNumbaNode(csp::Engine *engine, - void *stateObject, CallbackType numbaInitCallback, CallbackType numbaImplCallback, - PyObjectPtr inputs, - PyObjectPtr outputs, - NodeDef def, - PyObject *dataReference) : csp::Node(def, engine), - m_stateObject(stateObject), - m_numbaInitCallback(numbaInitCallback), - m_numbaImplCallback(numbaImplCallback), - m_dataReference(PyObjectPtr::incref(dataReference)) +PyNumbaNode::PyNumbaNode( csp::Engine * engine, void * stateObject, CallbackType numbaInitCallback, + CallbackType numbaImplCallback, PyObjectPtr inputs, PyObjectPtr outputs, NodeDef def, + PyObject * dataReference ) + : csp::Node( def, engine ) + , m_stateObject( stateObject ) + , m_numbaInitCallback( numbaInitCallback ) + , m_numbaImplCallback( numbaImplCallback ) + , m_dataReference( PyObjectPtr::incref( dataReference ) ) { - DEBUG_PRINT_WITH_LOC("Created numba node: " << (int64_t) (this) << " stateObject=" << (int64_t) (stateObject) - << " numbaImplCallback=" - << (int64_t) numbaImplCallback); - init(inputs, outputs); + DEBUG_PRINT_WITH_LOC( "Created numba node: " << (int64_t)( this ) << " stateObject=" << (int64_t)( stateObject ) + << " numbaImplCallback=" << (int64_t)numbaImplCallback ); + init( inputs, outputs ); } -PyNumbaNode::~PyNumbaNode() -{ -} +PyNumbaNode::~PyNumbaNode() {} -void PyNumbaNode::init(PyObjectPtr inputs, PyObjectPtr outputs) +void PyNumbaNode::init( PyObjectPtr inputs, PyObjectPtr outputs ) { - for (int input_idx = 0; input_idx < numInputs(); ++input_idx) + for( int input_idx = 0; input_idx < numInputs(); ++input_idx ) { - PyObject * inputType = PyTuple_GET_ITEM(inputs.ptr(), input_idx); + PyObject * inputType = PyTuple_GET_ITEM( inputs.ptr(), input_idx ); - //if basket - if (PyTuple_Check(inputType)) + // if basket + if( PyTuple_Check( inputType ) ) { - PyObject * shape = PyTuple_GET_ITEM(inputType, 0); + PyObject * shape = PyTuple_GET_ITEM( inputType, 0 ); std::uint64_t basketSize; - if (PyLong_Check(shape)) - basketSize = fromPython(shape); + if( PyLong_Check( shape ) ) + basketSize = fromPython( shape ); else { - if (!PyList_Check(shape)) - CSP_THROW(TypeError, "Expected basket type as int or list, got " << Py_TYPE(shape)->tp_name); - basketSize = PyList_GET_SIZE(shape); + if( !PyList_Check( shape ) ) + CSP_THROW( TypeError, "Expected basket type as int or list, got " << Py_TYPE( shape )->tp_name ); + basketSize = PyList_GET_SIZE( shape ); } - initInputBasket(input_idx, basketSize, false); + initInputBasket( input_idx, basketSize, false ); } } } -void PyNumbaNode::start() -{ - m_numbaInitCallback(this, m_stateObject); -} +void PyNumbaNode::start() { m_numbaInitCallback( this, m_stateObject ); } void PyNumbaNode::stop() { -// PyObjectPtr rv = PyObjectPtr::own(PyObject_CallMethod(m_gen.ptr(), "close", nullptr)); -// if (!rv.ptr()) -// CSP_THROW(PythonPassthrough, ""); + // PyObjectPtr rv = PyObjectPtr::own(PyObject_CallMethod(m_gen.ptr(), "close", nullptr)); + // if (!rv.ptr()) + // CSP_THROW(PythonPassthrough, ""); } -PyNumbaNode *PyNumbaNode::create(PyEngine *pyengine, PyObject *inputs, PyObject *outputs, - PyObject *stateObject, PyObject *numbaInitCallback, PyObject *numbaImplCallback, - PyObject *dataReference) +PyNumbaNode * PyNumbaNode::create( PyEngine * pyengine, PyObject * inputs, PyObject * outputs, PyObject * stateObject, + PyObject * numbaInitCallback, PyObject * numbaImplCallback, + PyObject * dataReference ) { - //parse inputs/outputs, create outputs time series, etc - Py_ssize_t numInputs = PyTuple_GET_SIZE(inputs); - Py_ssize_t numOutputs = PyTuple_GET_SIZE(outputs); + // parse inputs/outputs, create outputs time series, etc + Py_ssize_t numInputs = PyTuple_GET_SIZE( inputs ); + Py_ssize_t numOutputs = PyTuple_GET_SIZE( outputs ); - static_assert(sizeof(void *) == sizeof(int64_t)); - - void *stateObjectPtr = reinterpret_cast(fromPython(stateObject)); - CallbackType numbaInitCallbackPtr = reinterpret_cast(fromPython(numbaInitCallback)); - CallbackType numbaImplCallbackPtr = reinterpret_cast(fromPython(numbaImplCallback)); + static_assert( sizeof( void * ) == sizeof( int64_t ) ); + void * stateObjectPtr = reinterpret_cast( fromPython( stateObject ) ); + CallbackType numbaInitCallbackPtr = reinterpret_cast( fromPython( numbaInitCallback ) ); + CallbackType numbaImplCallbackPtr = reinterpret_cast( fromPython( numbaImplCallback ) ); if( size_t( numInputs ) > InputId::maxBasketElements() ) CSP_THROW( ValueError, "number of inputs exceeds limit of " << InputId::maxBasketElements() ); @@ -109,14 +105,12 @@ PyNumbaNode *PyNumbaNode::create(PyEngine *pyengine, PyObject *inputs, PyObject if( size_t( numOutputs ) > OutputId::maxBasketElements() ) CSP_THROW( ValueError, "number of outputs exceeds limit of " << OutputId::maxBasketElements() ); - - return pyengine->engine()->createOwnedObject(stateObjectPtr, numbaInitCallbackPtr, - numbaImplCallbackPtr, PyObjectPtr::incref(inputs), - PyObjectPtr::incref(outputs), - NodeDef(numInputs, numOutputs), dataReference); + return pyengine->engine()->createOwnedObject( + stateObjectPtr, numbaInitCallbackPtr, numbaImplCallbackPtr, PyObjectPtr::incref( inputs ), + PyObjectPtr::incref( outputs ), NodeDef( numInputs, numOutputs ), dataReference ); } -const char *PyNumbaNode::name() const +const char * PyNumbaNode::name() const { // TODO: propagate name from python here? return "PyNumbaNode"; @@ -124,114 +118,100 @@ const char *PyNumbaNode::name() const void PyNumbaNode::call_callback() { - m_numbaImplCallback(this, m_stateObject); -// if (!m_gen->ob_type->tp_iternext(m_gen.ptr())) -// CSP_THROW(PythonPassthrough, ""); + m_numbaImplCallback( this, m_stateObject ); + // if (!m_gen->ob_type->tp_iternext(m_gen.ptr())) + // CSP_THROW(PythonPassthrough, ""); } -void PyNumbaNode::executeImpl() -{ - call_callback(); -} +void PyNumbaNode::executeImpl() { call_callback(); } -static PyObject *PyNumbaNode_create(PyObject * module, PyObject * args) +static PyObject * PyNumbaNode_create( PyObject * module, PyObject * args ) { - CSP_BEGIN_METHOD ; - - PyEngine *engine; - PyObject * inputs; - PyObject * outputs; - PyObject * state; - PyObject * initCallback; - PyObject * implCallback; - PyObject * dataReference; - - if (!PyArg_ParseTuple(args, "O!O!O!O!O!O!O!", - &PyEngine::PyType, &engine, - &PyTuple_Type, &inputs, - &PyTuple_Type, &outputs, - &PyLong_Type, &state, - &PyLong_Type, &initCallback, - &PyLong_Type, &implCallback, - &PyTuple_Type, &dataReference)) - CSP_THROW(PythonPassthrough, ""); - - auto node = PyNumbaNode::create(engine, inputs, outputs, state, initCallback, implCallback, dataReference); - return PyNodeWrapper::create(node); + CSP_BEGIN_METHOD; + + PyEngine * engine; + PyObject * inputs; + PyObject * outputs; + PyObject * state; + PyObject * initCallback; + PyObject * implCallback; + PyObject * dataReference; + + if( !PyArg_ParseTuple( args, "O!O!O!O!O!O!O!", &PyEngine::PyType, &engine, &PyTuple_Type, &inputs, &PyTuple_Type, + &outputs, &PyLong_Type, &state, &PyLong_Type, &initCallback, &PyLong_Type, &implCallback, + &PyTuple_Type, &dataReference ) ) + CSP_THROW( PythonPassthrough, "" ); + + auto node = PyNumbaNode::create( engine, inputs, outputs, state, initCallback, implCallback, dataReference ); + return PyNodeWrapper::create( node ); CSP_RETURN_NULL; } -REGISTER_MODULE_METHOD("PyNumbaNode", PyNumbaNode_create, METH_VARARGS, "PyNumbaNode"); +REGISTER_MODULE_METHOD( "PyNumbaNode", PyNumbaNode_create, METH_VARARGS, "PyNumbaNode" ); -} +} // namespace csp::python -#define IMPLEMENT_TYPE_GETTERS_AND_SETTERS(typ_name, c_typ_name) \ -c_typ_name __csp_get_node_ ## typ_name ## _value__(int64_t node_ptr, int32_t input_index) \ -{ \ -auto node = reinterpret_cast(node_ptr); \ -auto res = node->tsinput(input_index)->lastValueTyped(); \ -return res; \ -} \ -void __csp_return_ ## typ_name ## _value__(int64_t node_ptr, int32_t output_index, c_typ_name value) \ -{ \ - auto node = reinterpret_cast(node_ptr); \ - node->tsoutput(output_index)->outputTickTyped(node->cycleCount(), node->now(), value); \ -} +#define IMPLEMENT_TYPE_GETTERS_AND_SETTERS( typ_name, c_typ_name ) \ + c_typ_name __csp_get_node_##typ_name##_value__( int64_t node_ptr, int32_t input_index ) \ + { \ + auto node = reinterpret_cast( node_ptr ); \ + auto res = node->tsinput( input_index )->lastValueTyped(); \ + return res; \ + } \ + void __csp_return_##typ_name##_value__( int64_t node_ptr, int32_t output_index, c_typ_name value ) \ + { \ + auto node = reinterpret_cast( node_ptr ); \ + node->tsoutput( output_index )->outputTickTyped( node->cycleCount(), node->now(), value ); \ + } namespace { -inline csp::python::PyNumbaNode *getNodeFromInt(int64_t node_ptr) +inline csp::python::PyNumbaNode * getNodeFromInt( int64_t node_ptr ) { - return reinterpret_cast(node_ptr); + return reinterpret_cast( node_ptr ); } - -inline csp::TimeSeriesProvider *getNonConstInput(csp::python::PyNumbaNode *node, int32_t input_index) +inline csp::TimeSeriesProvider * getNonConstInput( csp::python::PyNumbaNode * node, int32_t input_index ) { - return const_cast(node->tsinput(input_index)); + return const_cast( node->tsinput( input_index ) ); } -} - - -extern "C" { - -int64_t __csp_create_datetime_nanoseconds__(int32_t year, int32_t month, int32_t day, - int32_t hour, int32_t minute, int32_t second, - int32_t nanosecond) -{ - return csp::DateTime(year, month, day, - hour, minute, second, - nanosecond).asNanoseconds(); -} +} // namespace -bool __csp_numba_node_ticked__(int64_t node_ptr, int32_t input_index) +extern "C" { - auto node = getNodeFromInt(node_ptr); - return node->inputTicked(input_index); -} -bool __csp_numba_node_valid__(int64_t node_ptr, int32_t input_index) -{ - auto node = getNodeFromInt(node_ptr); - return node->tsinput(input_index)->valid(); -} + int64_t __csp_create_datetime_nanoseconds__( int32_t year, int32_t month, int32_t day, int32_t hour, int32_t minute, + int32_t second, int32_t nanosecond ) + { + return csp::DateTime( year, month, day, hour, minute, second, nanosecond ).asNanoseconds(); + } -bool __csp_numba_node_make_passive__(int64_t node_ptr, int32_t input_index) -{ - auto node = getNodeFromInt(node_ptr); - return getNonConstInput(node, input_index)->removeConsumer(node, csp::InputId(input_index)); -} + bool __csp_numba_node_ticked__( int64_t node_ptr, int32_t input_index ) + { + auto node = getNodeFromInt( node_ptr ); + return node->inputTicked( input_index ); + } -bool __csp_numba_node_make_active__(int64_t node_ptr, int32_t input_index) -{ - auto node = getNodeFromInt(node_ptr); - return getNonConstInput(node, input_index)->addConsumer(node, csp::InputId(input_index), true); -} + bool __csp_numba_node_valid__( int64_t node_ptr, int32_t input_index ) + { + auto node = getNodeFromInt( node_ptr ); + return node->tsinput( input_index )->valid(); + } + bool __csp_numba_node_make_passive__( int64_t node_ptr, int32_t input_index ) + { + auto node = getNodeFromInt( node_ptr ); + return getNonConstInput( node, input_index )->removeConsumer( node, csp::InputId( input_index ) ); + } -IMPLEMENT_TYPE_GETTERS_AND_SETTERS(int64, int64_t) -IMPLEMENT_TYPE_GETTERS_AND_SETTERS(double, double) -IMPLEMENT_TYPE_GETTERS_AND_SETTERS(bool, bool) + bool __csp_numba_node_make_active__( int64_t node_ptr, int32_t input_index ) + { + auto node = getNodeFromInt( node_ptr ); + return getNonConstInput( node, input_index )->addConsumer( node, csp::InputId( input_index ), true ); + } + IMPLEMENT_TYPE_GETTERS_AND_SETTERS( int64, int64_t ) + IMPLEMENT_TYPE_GETTERS_AND_SETTERS( double, double ) + IMPLEMENT_TYPE_GETTERS_AND_SETTERS( bool, bool ) } diff --git a/cpp/csp/python/PyNumpyAdapter.cpp b/cpp/csp/python/PyNumpyAdapter.cpp index cf08da83..3809085b 100644 --- a/cpp/csp/python/PyNumpyAdapter.cpp +++ b/cpp/csp/python/PyNumpyAdapter.cpp @@ -1,4 +1,4 @@ -//this is included first so that we do include without NO_IMPORT_ARRAY defined, which is done in NumpyInputAdapter.h +// this is included first so that we do include without NO_IMPORT_ARRAY defined, which is done in NumpyInputAdapter.h #include // // BUT! really we can't have this here and in NumpyConversions.cpp.... @@ -10,42 +10,36 @@ #include #include - namespace csp::python { static bool numpy_initialized = false; -static InputAdapter * numpy_adapter_creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, PushMode pushMode, PyObject * args ) +static InputAdapter * numpy_adapter_creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, + PushMode pushMode, PyObject * args ) { if( !numpy_initialized ) { - import_array() - numpy_initialized = true; - } + import_array() numpy_initialized = true; + } PyObject * type; PyArrayObject * pyDatetimes = nullptr; PyArrayObject * pyValues = nullptr; - if( !PyArg_ParseTuple( args, "OO!O!", - &type, - &PyArray_Type, &pyDatetimes, - &PyArray_Type, &pyValues ) ) + if( !PyArg_ParseTuple( args, "OO!O!", &type, &PyArray_Type, &pyDatetimes, &PyArray_Type, &pyValues ) ) CSP_THROW( PythonPassthrough, "" ); auto cspType = pyTypeAsCspType( pyType ); return switchCspType( cspType, - [ engine = pyengine -> engine(), &cspType, pyDatetimes, pyValues ]( - auto tag ) -> InputAdapter * + [engine = pyengine->engine(), &cspType, pyDatetimes, pyValues]( auto tag ) -> InputAdapter * { - using T = typename decltype(tag)::type; - return engine -> createOwnedObject>( - cspType, pyDatetimes, pyValues ); + using T = typename decltype( tag )::type; + return engine->createOwnedObject>( cspType, pyDatetimes, pyValues ); } ); } REGISTER_INPUT_ADAPTER( _npcurve, numpy_adapter_creator ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyOutputAdapter.cpp b/cpp/csp/python/PyOutputAdapter.cpp index 85aede0f..8e94b9e8 100644 --- a/cpp/csp/python/PyOutputAdapter.cpp +++ b/cpp/csp/python/PyOutputAdapter.cpp @@ -4,9 +4,9 @@ #include #include #include +#include #include #include -#include namespace csp::python { @@ -14,39 +14,32 @@ namespace csp::python class PyOutputAdapter : public OutputAdapter { public: - PyOutputAdapter( - Engine * engine, - PyObjectPtr pyadapter - ) - : OutputAdapter( engine ), - m_pyadapter( pyadapter ) - {} + PyOutputAdapter( Engine * engine, PyObjectPtr pyadapter ) + : OutputAdapter( engine ) + , m_pyadapter( pyadapter ) + { + } void executeImpl() override; void start() override; void stop() override; - + const char * name() const override { return "PyOutputAdapter"; } + private: PyObjectPtr m_pyadapter; }; void PyOutputAdapter::executeImpl() { - const TimeSeriesProvider *inputs = input(); - - PyObject* last_time = toPython(inputs->lastTime()); - PyObject* last_value = lastValueToPython(inputs); - - PyObjectPtr rv = PyObjectPtr::own( - PyObject_CallMethod( - m_pyadapter.ptr(), - "on_tick", - "OO", - PyObjectPtr::own(last_time).ptr(), - PyObjectPtr::own(last_value).ptr() - ) - ); + const TimeSeriesProvider * inputs = input(); + + PyObject * last_time = toPython( inputs->lastTime() ); + PyObject * last_value = lastValueToPython( inputs ); + + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "on_tick", "OO", + PyObjectPtr::own( last_time ).ptr(), + PyObjectPtr::own( last_value ).ptr() ) ); if( !rv.ptr() ) CSP_THROW( PythonPassthrough, "" ); } @@ -60,7 +53,7 @@ void PyOutputAdapter::start() void PyOutputAdapter::stop() { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "stop", nullptr ) ); + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "stop", nullptr ) ); if( !rv.ptr() ) CSP_THROW( PythonPassthrough, "" ); } @@ -68,17 +61,16 @@ void PyOutputAdapter::stop() static OutputAdapter * pyoutputadapter_creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * args ) { PyTypeObject * pyAdapterType = nullptr; - PyObject * adapterArgs = nullptr; - - if( !PyArg_ParseTuple( args, "O!O!", &PyType_Type, &pyAdapterType, - &PyTuple_Type, &adapterArgs ) ) + PyObject * adapterArgs = nullptr; + + if( !PyArg_ParseTuple( args, "O!O!", &PyType_Type, &pyAdapterType, &PyTuple_Type, &adapterArgs ) ) CSP_THROW( PythonPassthrough, "" ); - PyObject * pyAdapter = PyObject_Call( ( PyObject * ) pyAdapterType, adapterArgs, nullptr ); + PyObject * pyAdapter = PyObject_Call( (PyObject *)pyAdapterType, adapterArgs, nullptr ); if( !pyAdapter ) CSP_THROW( PythonPassthrough, "" ); - return pyengine -> engine() -> createOwnedObject( PyObjectPtr::own( pyAdapter ) ); + return pyengine->engine()->createOwnedObject( PyObjectPtr::own( pyAdapter ) ); } // NOTE: no python object is exported as its not needed @@ -86,4 +78,4 @@ static OutputAdapter * pyoutputadapter_creator( csp::AdapterManager * manager, P REGISTER_OUTPUT_ADAPTER( _outputadapter, pyoutputadapter_creator ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyOutputAdapterWrapper.cpp b/cpp/csp/python/PyOutputAdapterWrapper.cpp index fce3f06d..56d62d23 100644 --- a/cpp/csp/python/PyOutputAdapterWrapper.cpp +++ b/cpp/csp/python/PyOutputAdapterWrapper.cpp @@ -2,11 +2,11 @@ #include #include #include +#include #include #include #include #include -#include namespace csp::python { @@ -17,23 +17,20 @@ PyObject * PyOutputAdapterWrapper::createAdapter( Creator creator, PyObject * ar PyObject * pyAdapterManager = nullptr; - PyEngine * pyEngine = nullptr; - PyObject * pyArgs = nullptr; - + PyEngine * pyEngine = nullptr; + PyObject * pyArgs = nullptr; - if( !PyArg_ParseTuple( args, "OO!O!", - &pyAdapterManager, - &PyEngine::PyType, &pyEngine, - &PyTuple_Type, &pyArgs ) ) + if( !PyArg_ParseTuple( args, "OO!O!", &pyAdapterManager, &PyEngine::PyType, &pyEngine, &PyTuple_Type, &pyArgs ) ) CSP_THROW( PythonPassthrough, "" ); - csp::AdapterManager *adapterMgr = nullptr; + csp::AdapterManager * adapterMgr = nullptr; if( PyCapsule_CheckExact( pyAdapterManager ) ) adapterMgr = PyAdapterManagerWrapper::extractAdapterManager( pyAdapterManager ); auto adapter = creator( adapterMgr, pyEngine, pyArgs ); - PyOutputAdapterWrapper * object = ( PyOutputAdapterWrapper * ) PyOutputAdapterWrapper::PyType.tp_alloc( &PyOutputAdapterWrapper::PyType, 0 ); + PyOutputAdapterWrapper * object + = (PyOutputAdapterWrapper *)PyOutputAdapterWrapper::PyType.tp_alloc( &PyOutputAdapterWrapper::PyType, 0 ); new( object ) PyOutputAdapterWrapper( adapter ); return object; @@ -46,83 +43,84 @@ static PyObject * PyOutputAdapterWrapper_linkFrom( PyOutputAdapterWrapper * self int outputIdx, outputBasketIdx, inputIdx, inputBasketIdx; PyObject * source; - if( !PyArg_ParseTuple( args, "Oiiii", &source, - &outputIdx, &outputBasketIdx, - &inputIdx, &inputBasketIdx ) ) + if( !PyArg_ParseTuple( args, "Oiiii", &source, &outputIdx, &outputBasketIdx, &inputIdx, &inputBasketIdx ) ) return nullptr; OutputId outputId( outputIdx, outputBasketIdx ); if( PyType_IsSubtype( Py_TYPE( source ), &PyNodeWrapper::PyType ) ) { - self -> adapter() -> link( static_cast( source ) -> node() -> output( outputId ) ); + self->adapter()->link( static_cast( source )->node()->output( outputId ) ); } else if( PyType_IsSubtype( Py_TYPE( source ), &PyInputAdapterWrapper::PyType ) ) { - self -> adapter() -> link( static_cast( source ) -> adapter() ); + self->adapter()->link( static_cast( source )->adapter() ); } else - CSP_THROW( TypeError, "link_from expected PyNode or PyInputAdapter as source, got " << Py_TYPE( source ) -> tp_name ); + CSP_THROW( TypeError, + "link_from expected PyNode or PyInputAdapter as source, got " << Py_TYPE( source )->tp_name ); CSP_RETURN_NONE; } -static PyMethodDef PyOutputAdapterWrapper_methods[] = { - { "link_from", (PyCFunction) PyOutputAdapterWrapper_linkFrom, METH_VARARGS, "links node's output to target output adapter" }, - {NULL} -}; +static PyMethodDef PyOutputAdapterWrapper_methods[] + = { { "link_from", (PyCFunction)PyOutputAdapterWrapper_linkFrom, METH_VARARGS, + "links node's output to target output adapter" }, + { NULL } }; PyTypeObject PyOutputAdapterWrapper::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyOutputAdapterWrapper", /* tp_name */ - sizeof(PyOutputAdapterWrapper), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp output adapter wrapper", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyOutputAdapterWrapper_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ /* Low-level free-memory routine */ - 0, /* tp_is_gc */ /* For PyObject_IS_GC */ - 0, /* tp_bases */ - 0, /* tp_mro */ /* method resolution order */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0 /* tp_finalize */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyOutputAdapterWrapper", /* tp_name */ + sizeof( PyOutputAdapterWrapper ), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp output adapter wrapper", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyOutputAdapterWrapper_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + 0, + /* tp_free */ /* Low-level free-memory routine */ + 0, + /* tp_is_gc */ /* For PyObject_IS_GC */ + 0, /* tp_bases */ + 0, + /* tp_mro */ /* method resolution order */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0 /* tp_finalize */ }; REGISTER_TYPE_INIT( &PyOutputAdapterWrapper::PyType, "PyOutputAdapterWrapper" ) -} +} // namespace csp::python diff --git a/cpp/csp/python/PyOutputProxy.cpp b/cpp/csp/python/PyOutputProxy.cpp index 2db235bf..a71583c7 100644 --- a/cpp/csp/python/PyOutputProxy.cpp +++ b/cpp/csp/python/PyOutputProxy.cpp @@ -6,15 +6,16 @@ namespace csp::python { -PyOutputProxy::PyOutputProxy( PyObject * pyType, Node * node, OutputId id ) : m_pyType( PyObjectPtr::incref( pyType ) ), - m_node( node ), - m_id( id ) +PyOutputProxy::PyOutputProxy( PyObject * pyType, Node * node, OutputId id ) + : m_pyType( PyObjectPtr::incref( pyType ) ) + , m_node( node ) + , m_id( id ) { } PyOutputProxy * PyOutputProxy::create( PyObject * pyType, Node * node, OutputId id ) { - PyOutputProxy * proxy = ( PyOutputProxy * ) PyType.tp_new( &PyType, nullptr, nullptr ); + PyOutputProxy * proxy = (PyOutputProxy *)PyType.tp_new( &PyType, nullptr, nullptr ); new( proxy ) PyOutputProxy( pyType, node, id ); return proxy; } @@ -23,8 +24,8 @@ static void PyOutputProxy_dealloc( PyOutputProxy * self ) { CSP_BEGIN_METHOD; - ( self ) -> ~PyOutputProxy(); - Py_TYPE( self ) -> tp_free( self ); + ( self )->~PyOutputProxy(); + Py_TYPE( self )->tp_free( self ); CSP_RETURN; } @@ -33,83 +34,85 @@ static PyObject * PyOutputProxy_output( PyOutputProxy * proxy, PyObject * value { CSP_BEGIN_METHOD; - proxy -> outputTick( value ); + proxy->outputTick( value ); CSP_RETURN_NONE; } static PyNumberMethods PyOutputProxy_NumberMethods = { - (binaryfunc ) PyOutputProxy_output, /* binaryfunc nb_add */ - 0, /* binaryfunc nb_subtract */ - 0, /* binaryfunc nb_multiply */ - 0, /* binaryfunc nb_remainder */ - 0, /* binaryfunc nb_divmod */ - 0, /* ternaryfunc nb_power */ - 0, /* unaryfunc nb_negative */ - 0, /* unaryfunc nb_positive */ - 0, /* unaryfunc nb_absolute */ - 0, /* inquiry nb_nonzero */ - 0, /* unaryfunc nb_invert */ - 0, /* binaryfunc nb_lshift */ - 0, /* binaryfunc nb_rshift */ - 0, /* binaryfunc nb_and */ - 0, /* binaryfunc nb_xor */ - 0, /* binaryfunc nb_or */ - 0, /* unaryfunc nb_int */ - 0, /* void * reserved */ - 0 /* unaryfunc nb_float */ + (binaryfunc)PyOutputProxy_output, /* binaryfunc nb_add */ + 0, /* binaryfunc nb_subtract */ + 0, /* binaryfunc nb_multiply */ + 0, /* binaryfunc nb_remainder */ + 0, /* binaryfunc nb_divmod */ + 0, /* ternaryfunc nb_power */ + 0, /* unaryfunc nb_negative */ + 0, /* unaryfunc nb_positive */ + 0, /* unaryfunc nb_absolute */ + 0, /* inquiry nb_nonzero */ + 0, /* unaryfunc nb_invert */ + 0, /* binaryfunc nb_lshift */ + 0, /* binaryfunc nb_rshift */ + 0, /* binaryfunc nb_and */ + 0, /* binaryfunc nb_xor */ + 0, /* binaryfunc nb_or */ + 0, /* unaryfunc nb_int */ + 0, /* void * reserved */ + 0 /* unaryfunc nb_float */ }; PyTypeObject PyOutputProxy::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyOutputProxy", /* tp_name */ - sizeof(PyOutputProxy), /* tp_basicsize */ - 0, /* tp_itemsize */ - ( destructor ) PyOutputProxy_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - &PyOutputProxy_NumberMethods,/* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp output proxy", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ /* Low-level free-memory routine */ - 0, /* tp_is_gc */ /* For PyObject_IS_GC */ - 0, /* tp_bases */ - 0, /* tp_mro */ /* method resolution order */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0 /* tp_finalize */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyOutputProxy", /* tp_name */ + sizeof( PyOutputProxy ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyOutputProxy_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + &PyOutputProxy_NumberMethods, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp output proxy", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + 0, + /* tp_free */ /* Low-level free-memory routine */ + 0, + /* tp_is_gc */ /* For PyObject_IS_GC */ + 0, /* tp_bases */ + 0, + /* tp_mro */ /* method resolution order */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0 /* tp_finalize */ }; REGISTER_TYPE_INIT( &PyOutputProxy::PyType, "PyOutputProxy" ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyPullInputAdapter.cpp b/cpp/csp/python/PyPullInputAdapter.cpp index 2db555c5..a9589c46 100644 --- a/cpp/csp/python/PyPullInputAdapter.cpp +++ b/cpp/csp/python/PyPullInputAdapter.cpp @@ -1,8 +1,8 @@ #include #include #include -#include #include +#include #include namespace csp::python @@ -12,16 +12,16 @@ template class PyPullInputAdapter : public PullInputAdapter { public: - PyPullInputAdapter( Engine * engine, PyObjectPtr pyadapter, - PyObject * pyType, PushMode pushMode ) : PullInputAdapter( engine, pyTypeAsCspType( pyType ), pushMode ), - m_pyadapter( pyadapter ), - m_pyType( PyObjectPtr::incref( ( PyObject * ) pyType ) ) + PyPullInputAdapter( Engine * engine, PyObjectPtr pyadapter, PyObject * pyType, PushMode pushMode ) + : PullInputAdapter( engine, pyTypeAsCspType( pyType ), pushMode ) + , m_pyadapter( pyadapter ) + , m_pyType( PyObjectPtr::incref( (PyObject *)pyType ) ) { } void start( DateTime start, DateTime end ) override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "start", "OO", + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "start", "OO", PyObjectPtr::own( toPython( start ) ).ptr(), PyObjectPtr::own( toPython( end ) ).ptr() ) ); if( !rv.ptr() ) @@ -32,7 +32,7 @@ class PyPullInputAdapter : public PullInputAdapter void stopAdapter() override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "stop", nullptr ) ); + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "stop", nullptr ) ); if( !rv.ptr() ) CSP_THROW( PythonPassthrough, "" ); @@ -40,40 +40,40 @@ class PyPullInputAdapter : public PullInputAdapter bool next( DateTime & time, T & value ) override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "next", nullptr ) ); + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "next", nullptr ) ); if( !rv.ptr() ) { if( PyErr_Occurred() == PyExc_KeyboardInterrupt ) { - this -> rootEngine() -> shutdown(); + this->rootEngine()->shutdown(); return false; } CSP_THROW( PythonPassthrough, "" ); } - + if( rv.ptr() == Py_None ) return false; if( !PyTuple_Check( rv.ptr() ) || PyTuple_GET_SIZE( rv.ptr() ) != 2 ) CSP_THROW( TypeError, "PyPullInputAdapter::next expects None or ( datetime, value ), got " << rv ); - time = fromPython( PyTuple_GET_ITEM( rv.ptr(), 0 ) ); + time = fromPython( PyTuple_GET_ITEM( rv.ptr(), 0 ) ); PyObject * pyValue = PyTuple_GET_ITEM( rv.ptr(), 1 ); try { - if( !validatePyType( this -> dataType(), m_pyType.ptr(), pyValue ) ) + if( !validatePyType( this->dataType(), m_pyType.ptr(), pyValue ) ) CSP_THROW( TypeError, "" ); - value = fromPython( pyValue, *this -> dataType() ); + value = fromPython( pyValue, *this->dataType() ); } catch( TypeError & e ) { - CSP_THROW( TypeError, "\"" << Py_TYPE( m_pyadapter.ptr() ) -> tp_name << "\" pull adapter expected output type to be of type \"" - << pyTypeToString( m_pyType.ptr() ) << "\" got type \"" << Py_TYPE( pyValue ) -> tp_name << "\"" ); - + CSP_THROW( + TypeError, + "\"" << Py_TYPE( m_pyadapter.ptr() )->tp_name << "\" pull adapter expected output type to be of type \"" + << pyTypeToString( m_pyType.ptr() ) << "\" got type \"" << Py_TYPE( pyValue )->tp_name << "\"" ); } return true; - } private: @@ -81,31 +81,30 @@ class PyPullInputAdapter : public PullInputAdapter PyObjectPtr m_pyType; }; -static InputAdapter * pypulladapter_creator( csp::AdapterManager * manager, PyEngine * pyengine, - PyObject * pyType, PushMode pushMode, PyObject * args ) +static InputAdapter * pypulladapter_creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, + PushMode pushMode, PyObject * args ) { PyTypeObject * pyAdapterType = nullptr; - PyObject * adapterArgs = nullptr; - if( !PyArg_ParseTuple( args, "O!O!", &PyType_Type, &pyAdapterType, - &PyTuple_Type, &adapterArgs ) ) + PyObject * adapterArgs = nullptr; + if( !PyArg_ParseTuple( args, "O!O!", &PyType_Type, &pyAdapterType, &PyTuple_Type, &adapterArgs ) ) CSP_THROW( PythonPassthrough, "" ); - PyObject * pyAdapter = PyObject_Call( ( PyObject * ) pyAdapterType, adapterArgs, nullptr ); + PyObject * pyAdapter = PyObject_Call( (PyObject *)pyAdapterType, adapterArgs, nullptr ); if( !pyAdapter ) CSP_THROW( PythonPassthrough, "" ); - return switchPyType( pyType, - [engine = pyengine -> engine(), pyAdapter,pyType,pushMode] ( auto tag ) -> InputAdapter * - { - using T = typename decltype(tag)::type; - return engine -> createOwnedObject>( PyObjectPtr::own( pyAdapter ), pyType, pushMode ); + return switchPyType( pyType, + [engine = pyengine->engine(), pyAdapter, pyType, pushMode]( auto tag ) -> InputAdapter * + { + using T = typename decltype( tag )::type; + return engine->createOwnedObject>( PyObjectPtr::own( pyAdapter ), + pyType, pushMode ); } ); } - // NOTE: no python object is exported as its not needed // at this time. REGISTER_INPUT_ADAPTER( _pulladapter, pypulladapter_creator ); -} +} // namespace csp::python diff --git a/cpp/csp/python/PyPushInputAdapter.cpp b/cpp/csp/python/PyPushInputAdapter.cpp index 3b5f91b0..9bbc9801 100644 --- a/cpp/csp/python/PyPushInputAdapter.cpp +++ b/cpp/csp/python/PyPushInputAdapter.cpp @@ -1,10 +1,10 @@ #include #include #include -#include -#include #include +#include #include +#include namespace csp::python { @@ -16,19 +16,19 @@ static int PushBatch_init( PyPushBatch * self, PyObject * args, PyObject * kwarg CSP_THROW( TypeError, "PushBatch expected engine as single positional argument" ); PyObject * pyengine = PyTuple_GET_ITEM( args, 0 ); - if( pyengine -> ob_type != &PyEngine::PyType ) + if( pyengine->ob_type != &PyEngine::PyType ) CSP_THROW( TypeError, "PushBatch expected engine as single positional argument" ); - csp::RootEngine * engine = ( ( PyEngine * ) pyengine ) -> rootEngine(); - new( &self -> batch ) PushBatch( engine ); + csp::RootEngine * engine = ( (PyEngine *)pyengine )->rootEngine(); + new( &self->batch ) PushBatch( engine ); CSP_RETURN_INT; } static void PushBatch_dealloc( PyPushBatch * self ) { CSP_BEGIN_METHOD; - self -> batch.~PushBatch(); - Py_TYPE( self ) -> tp_free( self ); + self->batch.~PushBatch(); + Py_TYPE( self )->tp_free( self ); CSP_RETURN; } @@ -42,56 +42,54 @@ static PyObject * PushBatch_exit( PyPushBatch * self, PyObject * args ) { PyObject * exception = PyTuple_GET_ITEM( args, 0 ); if( exception != Py_None ) - self -> batch.clear(); + self->batch.clear(); - //We flush here since we want to flush on exit from scope, not when object is destroyed ( which can outlast the scope! ) - self -> batch.flush(); - Py_RETURN_NONE; + // We flush here since we want to flush on exit from scope, not when object is destroyed ( which can outlast the + // scope! ) + self->batch.flush(); + Py_RETURN_NONE; } -static PyMethodDef PushBatch_methods[] = { - { "__enter__", (PyCFunction) PushBatch_enter, METH_NOARGS, "enter context" }, - { "__exit__", (PyCFunction) PushBatch_exit, METH_VARARGS, "exit context" }, - {NULL} -}; +static PyMethodDef PushBatch_methods[] = { { "__enter__", (PyCFunction)PushBatch_enter, METH_NOARGS, "enter context" }, + { "__exit__", (PyCFunction)PushBatch_exit, METH_VARARGS, "exit context" }, + { NULL } }; PyTypeObject PyPushBatch::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PushBatch", /* tp_name */ - sizeof(PyPushBatch), /* tp_basicsize */ - 0, /* tp_itemsize */ - ( destructor ) PushBatch_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "csp push batch", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PushBatch_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - ( initproc ) PushBatch_init, /* tp_init */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PushBatch", /* tp_name */ + sizeof( PyPushBatch ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PushBatch_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "csp push batch", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PushBatch_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)PushBatch_init, /* tp_init */ 0, PyType_GenericNew, }; @@ -99,17 +97,17 @@ PyTypeObject PyPushBatch::PyType = { class PyPushInputAdapter : public PushInputAdapter { public: - PyPushInputAdapter( Engine * engine, AdapterManager * manager, - PyObjectPtr pyadapter, PyObject * pyType, - PushMode pushMode, PushGroup * pushGroup ) : PushInputAdapter( engine, pyTypeAsCspType( pyType ), pushMode, pushGroup ), - m_pyadapter( pyadapter ), - m_pyType( PyObjectPtr::incref( ( PyObject * ) pyType ) ) + PyPushInputAdapter( Engine * engine, AdapterManager * manager, PyObjectPtr pyadapter, PyObject * pyType, + PushMode pushMode, PushGroup * pushGroup ) + : PushInputAdapter( engine, pyTypeAsCspType( pyType ), pushMode, pushGroup ) + , m_pyadapter( pyadapter ) + , m_pyType( PyObjectPtr::incref( (PyObject *)pyType ) ) { } void start( DateTime start, DateTime end ) override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "start", "OO", + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "start", "OO", PyObjectPtr::own( toPython( start ) ).ptr(), PyObjectPtr::own( toPython( end ) ).ptr() ) ); if( !rv.ptr() ) @@ -118,7 +116,7 @@ class PyPushInputAdapter : public PushInputAdapter void stop() override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "stop", nullptr ) ); + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "stop", nullptr ) ); if( !rv.ptr() ) { @@ -145,8 +143,8 @@ class TypedPyPushInputAdapter : public PyPushInputAdapter { public: TypedPyPushInputAdapter( Engine * engine, AdapterManager * manager, PyObjectPtr pyadapter, PyObject * pyType, - PushMode pushMode, PushGroup * pushGroup ): - PyPushInputAdapter( engine, manager, pyadapter, pyType, pushMode, pushGroup ) + PushMode pushMode, PushGroup * pushGroup ) + : PyPushInputAdapter( engine, manager, pyadapter, pyType, pushMode, pushGroup ) { } @@ -154,46 +152,48 @@ class TypedPyPushInputAdapter : public PyPushInputAdapter { try { - if( !validatePyType( this -> dataType(), m_pyType.ptr(), value ) ) + if( !validatePyType( this->dataType(), m_pyType.ptr(), value ) ) CSP_THROW( TypeError, "" ); - pushTick( std::move(fromPython( value, *this -> dataType() )), batch ); + pushTick( std::move( fromPython( value, *this->dataType() ) ), batch ); } catch( const TypeError & ) { - CSP_THROW( TypeError, "\"" << Py_TYPE( m_pyadapter.ptr() ) -> tp_name << "\" push adapter expected output type to be of type \"" - << pyTypeToString( m_pyType.ptr() ) << "\" got type \"" << Py_TYPE( value ) -> tp_name << "\"" ); + CSP_THROW( + TypeError, + "\"" << Py_TYPE( m_pyadapter.ptr() )->tp_name << "\" push adapter expected output type to be of type \"" + << pyTypeToString( m_pyType.ptr() ) << "\" got type \"" << Py_TYPE( value )->tp_name << "\"" ); } } }; struct PyPushInputAdapter_PyObject { - PyObject_HEAD - PyPushInputAdapter * adapter; + PyObject_HEAD PyPushInputAdapter * adapter; - static PyObject * pushTick( PyPushInputAdapter_PyObject * self, PyObject * args, PyObject **kwargs ) + static PyObject * pushTick( PyPushInputAdapter_PyObject * self, PyObject * args, PyObject ** kwargs ) { CSP_BEGIN_METHOD; - PyObject *pyvalue = nullptr; - PyObject *pybatch = nullptr; + PyObject * pyvalue = nullptr; + PyObject * pybatch = nullptr; Py_ssize_t len = PyTuple_GET_SIZE( args ); if( len < 1 || len > 2 ) CSP_THROW( TypeError, "push_tick takes value and optional batch as positional arguments" ); pyvalue = PyTuple_GET_ITEM( args, 0 ); - + PushBatch * batch = nullptr; if( len == 2 ) { pybatch = PyTuple_GET_ITEM( args, 1 ); - if( pybatch -> ob_type != &PyPushBatch::PyType ) - CSP_THROW( TypeError, "push_tick expected PushBatch type as second argument, got " << pybatch -> ob_type -> tp_name ); + if( pybatch->ob_type != &PyPushBatch::PyType ) + CSP_THROW( TypeError, + "push_tick expected PushBatch type as second argument, got " << pybatch->ob_type->tp_name ); - batch = &( ( PyPushBatch * ) pybatch ) -> batch; + batch = &( (PyPushBatch *)pybatch )->batch; } - self -> adapter -> pushPyTick( pyvalue, batch ); + self->adapter->pushPyTick( pyvalue, batch ); CSP_RETURN_NONE; } @@ -201,94 +201,95 @@ struct PyPushInputAdapter_PyObject static PyTypeObject PyType; }; -static PyMethodDef PyPushInputAdapter_PyObject_methods[] = { - { "push_tick", (PyCFunction) PyPushInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, - {NULL} -}; +static PyMethodDef PyPushInputAdapter_PyObject_methods[] + = { { "push_tick", (PyCFunction)PyPushInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, { NULL } }; PyTypeObject PyPushInputAdapter_PyObject::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyPushInputAdapter", /* tp_name */ - sizeof(PyPushInputAdapter_PyObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - "csp push input adapter", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyPushInputAdapter_PyObject_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyPushInputAdapter", /* tp_name */ + sizeof( PyPushInputAdapter_PyObject ), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + "csp push input adapter", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyPushInputAdapter_PyObject_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ 0, PyType_GenericNew, }; -static InputAdapter * pypushinputadapter_creator( csp::AdapterManager * manager, PyEngine * pyengine, - PyObject * pyType, PushMode pushMode, PyObject * args ) +static InputAdapter * pypushinputadapter_creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, + PushMode pushMode, PyObject * args ) { PyTypeObject * pyAdapterType = nullptr; - PyObject * adapterArgs = nullptr; + PyObject * adapterArgs = nullptr; PyObject * pyPushGroup; if( !PyArg_ParseTuple( args, "O!OO!", &PyType_Type, &pyAdapterType, &pyPushGroup, &PyTuple_Type, &adapterArgs ) ) CSP_THROW( PythonPassthrough, "" ); if( !PyType_IsSubtype( pyAdapterType, &PyPushInputAdapter_PyObject::PyType ) ) - CSP_THROW( TypeError, "Expected PushInputAdapter derived type, got " << pyAdapterType -> tp_name ); + CSP_THROW( TypeError, "Expected PushInputAdapter derived type, got " << pyAdapterType->tp_name ); - csp::PushGroup *pushGroup = nullptr; + csp::PushGroup * pushGroup = nullptr; if( pyPushGroup != Py_None ) { - pushGroup = ( csp::PushGroup * ) PyCapsule_GetPointer( pyPushGroup, nullptr ); + pushGroup = (csp::PushGroup *)PyCapsule_GetPointer( pyPushGroup, nullptr ); if( !pushGroup ) { PyErr_Clear(); - CSP_THROW( TypeError, "Expected PushGroup instance for push group, got: " << PyObjectPtr::incref( pyPushGroup ) ); + CSP_THROW( TypeError, + "Expected PushGroup instance for push group, got: " << PyObjectPtr::incref( pyPushGroup ) ); } } - PyPushInputAdapter_PyObject * pyAdapter = ( PyPushInputAdapter_PyObject * ) PyObject_Call( ( PyObject * ) pyAdapterType, adapterArgs, nullptr ); + PyPushInputAdapter_PyObject * pyAdapter + = (PyPushInputAdapter_PyObject *)PyObject_Call( (PyObject *)pyAdapterType, adapterArgs, nullptr ); if( !pyAdapter ) CSP_THROW( PythonPassthrough, "" ); - switchPyType( pyType, - [&]( auto tag ) - { - pyAdapter -> adapter = pyengine -> engine() -> createOwnedObject>( - manager, PyObjectPtr::own( ( PyObject * ) pyAdapter ), pyType, pushMode, pushGroup ); - } ); - - return pyAdapter -> adapter; + switchPyType( + pyType, + [&]( auto tag ) + { + pyAdapter->adapter + = pyengine->engine()->createOwnedObject>( + manager, PyObjectPtr::own( (PyObject *)pyAdapter ), pyType, pushMode, pushGroup ); + } ); + + return pyAdapter->adapter; } -//PushGroup +// PushGroup static void destroy_push_group( PyObject * capsule ) { - delete ( ( PushGroup * ) PyCapsule_GetPointer( capsule, nullptr ) ); + delete( (PushGroup *)PyCapsule_GetPointer( capsule, nullptr ) ); } static PyObject * create_push_group( PyObject * module, PyObject * args ) @@ -296,12 +297,10 @@ static PyObject * create_push_group( PyObject * module, PyObject * args ) return PyCapsule_New( new PushGroup(), nullptr, destroy_push_group ); } - REGISTER_TYPE_INIT( &PyPushInputAdapter_PyObject::PyType, "PyPushInputAdapter" ); -REGISTER_TYPE_INIT( &PyPushBatch::PyType, "PushBatch" ); +REGISTER_TYPE_INIT( &PyPushBatch::PyType, "PushBatch" ); REGISTER_INPUT_ADAPTER( _pushadapter, pypushinputadapter_creator ); REGISTER_MODULE_METHOD( "PushGroup", create_push_group, METH_NOARGS, "PushGroup" ); -} - +} // namespace csp::python diff --git a/cpp/csp/python/PyPushPullInputAdapter.cpp b/cpp/csp/python/PyPushPullInputAdapter.cpp index d53ed972..d5b995cb 100644 --- a/cpp/csp/python/PyPushPullInputAdapter.cpp +++ b/cpp/csp/python/PyPushPullInputAdapter.cpp @@ -2,10 +2,10 @@ #include #include #include -#include -#include #include +#include #include +#include namespace csp::python { @@ -13,15 +13,15 @@ namespace csp::python class PyPushPullInputAdapter : public PushPullInputAdapter { public: - PyPushPullInputAdapter( Engine * engine, AdapterManager * manager, - PyObjectPtr pyadapter, PyObject * pyType, - PushMode pushMode, PushGroup * pushGroup ) : PushPullInputAdapter( engine, pyTypeAsCspType( pyType ), pushMode, pushGroup ), - m_pyadapter( pyadapter ), - m_pyType( PyObjectPtr::incref( ( PyObject * ) pyType ) ) + PyPushPullInputAdapter( Engine * engine, AdapterManager * manager, PyObjectPtr pyadapter, PyObject * pyType, + PushMode pushMode, PushGroup * pushGroup ) + : PushPullInputAdapter( engine, pyTypeAsCspType( pyType ), pushMode, pushGroup ) + , m_pyadapter( pyadapter ) + , m_pyType( PyObjectPtr::incref( (PyObject *)pyType ) ) { } - //override nextPullEvent so we can release GIL while we wait + // override nextPullEvent so we can release GIL while we wait PushPullInputAdapter::PullDataEvent * nextPullEvent() override { ReleaseGIL release; @@ -30,7 +30,7 @@ class PyPushPullInputAdapter : public PushPullInputAdapter void start( DateTime start, DateTime end ) override { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "start", "OO", + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "start", "OO", PyObjectPtr::own( toPython( start ) ).ptr(), PyObjectPtr::own( toPython( end ) ).ptr() ) ); if( !rv.ptr() ) @@ -43,7 +43,7 @@ class PyPushPullInputAdapter : public PushPullInputAdapter { PushPullInputAdapter::stop(); - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "stop", nullptr ) ); + PyObjectPtr rv = PyObjectPtr::own( PyObject_CallMethod( m_pyadapter.ptr(), "stop", nullptr ) ); if( !rv.ptr() ) CSP_THROW( PythonPassthrough, "" ); @@ -61,8 +61,8 @@ class TypedPyPushPullInputAdapter : public PyPushPullInputAdapter { public: TypedPyPushPullInputAdapter( Engine * engine, AdapterManager * manager, PyObjectPtr pyadapter, PyObject * pyType, - PushMode pushMode, PushGroup * pushGroup ): - PyPushPullInputAdapter( engine, manager, pyadapter, pyType, pushMode, pushGroup ) + PushMode pushMode, PushGroup * pushGroup ) + : PyPushPullInputAdapter( engine, manager, pyadapter, pyType, pushMode, pushGroup ) { } @@ -70,30 +70,32 @@ class TypedPyPushPullInputAdapter : public PyPushPullInputAdapter { try { - if( !validatePyType( this -> dataType(), m_pyType.ptr(), value ) ) + if( !validatePyType( this->dataType(), m_pyType.ptr(), value ) ) CSP_THROW( TypeError, "" ); - pushTick( live, fromPython( time ), std::move( fromPython( value, *this -> dataType() ) ), batch ); + pushTick( live, fromPython( time ), std::move( fromPython( value, *this->dataType() ) ), + batch ); } catch( const TypeError & ) { - CSP_THROW( TypeError, "\"" << Py_TYPE( m_pyadapter.ptr() ) -> tp_name << "\" push adapter expected output type to be of type \"" - << pyTypeToString( m_pyType.ptr() ) << "\" got type \"" << Py_TYPE( value ) -> tp_name << "\"" ); + CSP_THROW( + TypeError, + "\"" << Py_TYPE( m_pyadapter.ptr() )->tp_name << "\" push adapter expected output type to be of type \"" + << pyTypeToString( m_pyType.ptr() ) << "\" got type \"" << Py_TYPE( value )->tp_name << "\"" ); } } }; struct PyPushPullInputAdapter_PyObject { - PyObject_HEAD - PyPushPullInputAdapter * adapter; + PyObject_HEAD PyPushPullInputAdapter * adapter; - static PyObject * pushTick( PyPushPullInputAdapter_PyObject * self, PyObject * args, PyObject **kwargs ) + static PyObject * pushTick( PyPushPullInputAdapter_PyObject * self, PyObject * args, PyObject ** kwargs ) { CSP_BEGIN_METHOD; - PyObject *pylive; - PyObject *pytime; - PyObject *pyvalue; - PyObject *pybatch = nullptr; + PyObject * pylive; + PyObject * pytime; + PyObject * pyvalue; + PyObject * pybatch = nullptr; if( !PyArg_UnpackTuple( args, "push_tick", 3, 4, &pylive, &pytime, &pyvalue, &pybatch ) ) CSP_THROW( PythonPassthrough, "" ); @@ -101,111 +103,113 @@ struct PyPushPullInputAdapter_PyObject PushBatch * batch = nullptr; if( pybatch ) { - if( pybatch -> ob_type != &PyPushBatch::PyType ) - CSP_THROW( TypeError, "push_tick expected PushBatch type as second argument, got " << pybatch -> ob_type -> tp_name ); + if( pybatch->ob_type != &PyPushBatch::PyType ) + CSP_THROW( TypeError, + "push_tick expected PushBatch type as second argument, got " << pybatch->ob_type->tp_name ); - batch = &( ( PyPushBatch * ) pybatch ) -> batch; + batch = &( (PyPushBatch *)pybatch )->batch; } - self -> adapter -> pushPyTick( fromPython( pylive ), pytime, pyvalue, batch ); + self->adapter->pushPyTick( fromPython( pylive ), pytime, pyvalue, batch ); CSP_RETURN_NONE; } - - static PyObject * flagReplayComplete( PyPushPullInputAdapter_PyObject * self, PyObject * args, PyObject **kwargs ) + + static PyObject * flagReplayComplete( PyPushPullInputAdapter_PyObject * self, PyObject * args, PyObject ** kwargs ) { CSP_BEGIN_METHOD; - self -> adapter -> flagReplayComplete(); + self->adapter->flagReplayComplete(); CSP_RETURN_NONE; } static PyTypeObject PyType; }; -static PyMethodDef PyPushPullInputAdapter_PyObject_methods[] = { - { "push_tick", (PyCFunction) PyPushPullInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, - { "flag_replay_complete", (PyCFunction) PyPushPullInputAdapter_PyObject::flagReplayComplete, METH_VARARGS, "finish replay ticks" }, - {NULL} -}; +static PyMethodDef PyPushPullInputAdapter_PyObject_methods[] + = { { "push_tick", (PyCFunction)PyPushPullInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, + { "flag_replay_complete", (PyCFunction)PyPushPullInputAdapter_PyObject::flagReplayComplete, METH_VARARGS, + "finish replay ticks" }, + { NULL } }; PyTypeObject PyPushPullInputAdapter_PyObject::PyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_cspimpl.PyPushPullInputAdapter", /* tp_name */ - sizeof(PyPushPullInputAdapter_PyObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - "csp push input adapter", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyPushPullInputAdapter_PyObject_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ + PyVarObject_HEAD_INIT( NULL, 0 ) "_cspimpl.PyPushPullInputAdapter", /* tp_name */ + sizeof( PyPushPullInputAdapter_PyObject ), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + "csp push input adapter", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyPushPullInputAdapter_PyObject_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ 0, PyType_GenericNew, }; -static InputAdapter * pypushpullinputadapter_creator( csp::AdapterManager * manager, PyEngine * pyengine, +static InputAdapter * pypushpullinputadapter_creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, PushMode pushMode, PyObject * args ) { PyTypeObject * pyAdapterType = nullptr; - PyObject * adapterArgs = nullptr; + PyObject * adapterArgs = nullptr; PyObject * pyPushGroup; if( !PyArg_ParseTuple( args, "O!OO!", &PyType_Type, &pyAdapterType, &pyPushGroup, &PyTuple_Type, &adapterArgs ) ) CSP_THROW( PythonPassthrough, "" ); if( !PyType_IsSubtype( pyAdapterType, &PyPushPullInputAdapter_PyObject::PyType ) ) - CSP_THROW( TypeError, "Expected PushPullInputAdapter derived type, got " << pyAdapterType -> tp_name ); + CSP_THROW( TypeError, "Expected PushPullInputAdapter derived type, got " << pyAdapterType->tp_name ); - csp::PushGroup *pushGroup = nullptr; + csp::PushGroup * pushGroup = nullptr; if( pyPushGroup != Py_None ) { - pushGroup = ( csp::PushGroup * ) PyCapsule_GetPointer( pyPushGroup, nullptr ); + pushGroup = (csp::PushGroup *)PyCapsule_GetPointer( pyPushGroup, nullptr ); if( !pushGroup ) CSP_THROW( PythonPassthrough, "" ); } - PyPushPullInputAdapter_PyObject * pyAdapter = ( PyPushPullInputAdapter_PyObject * ) PyObject_Call( ( PyObject * ) pyAdapterType, adapterArgs, nullptr ); + PyPushPullInputAdapter_PyObject * pyAdapter + = (PyPushPullInputAdapter_PyObject *)PyObject_Call( (PyObject *)pyAdapterType, adapterArgs, nullptr ); if( !pyAdapter ) CSP_THROW( PythonPassthrough, "" ); - switchPyType( pyType, - [&]( auto tag ) - { - pyAdapter -> adapter = pyengine -> engine() -> createOwnedObject>( - manager, PyObjectPtr::own( ( PyObject * ) pyAdapter ), pyType, pushMode, pushGroup ); - } ); - - return pyAdapter -> adapter; + switchPyType( + pyType, + [&]( auto tag ) + { + pyAdapter->adapter + = pyengine->engine()->createOwnedObject>( + manager, PyObjectPtr::own( (PyObject *)pyAdapter ), pyType, pushMode, pushGroup ); + } ); + + return pyAdapter->adapter; } REGISTER_TYPE_INIT( &PyPushPullInputAdapter_PyObject::PyType, "PyPushPullInputAdapter" ); REGISTER_INPUT_ADAPTER( _pushpulladapter, pypushpullinputadapter_creator ); -} - +} // namespace csp::python diff --git a/cpp/csp/python/PyStruct.cpp b/cpp/csp/python/PyStruct.cpp index 24df48c6..73a78dcb 100644 --- a/cpp/csp/python/PyStruct.cpp +++ b/cpp/csp/python/PyStruct.cpp @@ -5,12 +5,15 @@ #include #include #include -#include #include -#include +#include #include +#include -namespace csp::python { class PyObjectStructField; } +namespace csp::python +{ +class PyObjectStructField; +} namespace csp::python { @@ -19,32 +22,33 @@ class PyObjectStructField final : public DialectGenericStructField { public: using BASE = DialectGenericStructField; - PyObjectStructField( const std::string & name, - PyTypeObjectPtr pytype ) : DialectGenericStructField( name, sizeof( PyObjectPtr ), alignof( PyObjectPtr ) ), - m_pytype( pytype ) - {} - + PyObjectStructField( const std::string & name, PyTypeObjectPtr pytype ) + : DialectGenericStructField( name, sizeof( PyObjectPtr ), alignof( PyObjectPtr ) ) + , m_pytype( pytype ) + { + } PyTypeObject * pytype() const { return m_pytype.ptr(); } void setValue( Struct * s, const DialectGenericType & obj ) const override { - auto& pyobj = reinterpret_cast(obj); - if( !PyObject_IsInstance( pyobj.ptr(), ( PyObject * ) m_pytype.ptr() ) ) - CSP_THROW( TypeError, "Invalid " << m_pytype -> tp_name << " type, expected " << m_pytype -> tp_name << " got " << - Py_TYPE( pyobj.ptr() ) -> tp_name << " for field '" << fieldname() << "'" ); + auto & pyobj = reinterpret_cast( obj ); + if( !PyObject_IsInstance( pyobj.ptr(), (PyObject *)m_pytype.ptr() ) ) + CSP_THROW( TypeError, + "Invalid " << m_pytype->tp_name << " type, expected " << m_pytype->tp_name << " got " + << Py_TYPE( pyobj.ptr() )->tp_name << " for field '" << fieldname() << "'" ); - BASE::setValue(s, obj); + BASE::setValue( s, obj ); } private: PyTypeObjectPtr m_pytype; }; -DialectStructMeta::DialectStructMeta( PyTypeObject * pyType, const std::string & name, - const Fields & flds, std::shared_ptr base ) : - StructMeta( name, flds, base ), - m_pyType( pyType ) +DialectStructMeta::DialectStructMeta( PyTypeObject * pyType, const std::string & name, const Fields & flds, + std::shared_ptr base ) + : StructMeta( name, flds, base ) + , m_pyType( pyType ) { } @@ -55,14 +59,16 @@ It took quite a while to get this straight. PyStructMeta::PyType is the StructM Instances of PyStructMeta::PyType are PyStrutMeta types, which are Struct class definitions. PyStructMeta types ( aka Struct class types ) are forced to inherit from PyStruct::PyType. This is done in python code where csp.Struct is defined as class Struct( _cspimpl.PyStruct, metaclass = StructMeta ). -StructMeta is defined in python and is a derived type of _cspimpl.PyStructMeta so that we can defined some helper code in python. +StructMeta is defined in python and is a derived type of _cspimpl.PyStructMeta so that we can defined some helper code +in python. This is a lot to take in... so again In python we have: class StructMeta( _cspimpl.PyStructMeta ): def __new__( ... ): #extract meta info - super().__new__( ..., metainfo ) <-- this calls into PyStructMeta_new in c++, the base type, which will return a new PyStructMeta instance ( a new struct type ) + super().__new__( ..., metainfo ) <-- this calls into PyStructMeta_new in c++, the base type, which will return +a new PyStructMeta instance ( a new struct type ) class Struct( _cspimpl.PyStruct, metaclass = StructMeta ): # python side methods... @@ -71,34 +77,33 @@ class MyStruct( Struct ): a = int ... -when MyStruct is declared, it will call StructMeta::__new__ which will call PyStructMeta_new and create the MyStruct type, which is PyStructMeta instance -when MyStruct() instance is created, it'll defer required calls to _cspimpl.PyStruct, its baseclass +when MyStruct is declared, it will call StructMeta::__new__ which will call PyStructMeta_new and create the MyStruct +type, which is PyStructMeta instance when MyStruct() instance is created, it'll defer required calls to +_cspimpl.PyStruct, its baseclass Hope this all makes sense when I read this in 3 months */ -static PyObject * PyStructMeta_new( PyTypeObject *subtype, PyObject *args, PyObject *kwds ) +static PyObject * PyStructMeta_new( PyTypeObject * subtype, PyObject * args, PyObject * kwds ) { CSP_BEGIN_METHOD; PyObject * pyname; PyObject * bases; PyObject * dict; - if( !PyArg_ParseTuple( args, "UO!O!", - &pyname, - &PyTuple_Type, &bases, - &PyDict_Type, &dict ) ) + if( !PyArg_ParseTuple( args, "UO!O!", &pyname, &PyTuple_Type, &bases, &PyDict_Type, &dict ) ) CSP_THROW( PythonPassthrough, "" ); - //subtype is python defined StructMeta class - PyStructMeta * pymeta = ( PyStructMeta * ) PyType_Type.tp_new( subtype, args, kwds ); + // subtype is python defined StructMeta class + PyStructMeta * pymeta = (PyStructMeta *)PyType_Type.tp_new( subtype, args, kwds ); - //Note that we call ctor without parents so as not to 0-init the base POD PyTypeObject class after its been initialized - new ( pymeta ) PyStructMeta; + // Note that we call ctor without parents so as not to 0-init the base POD PyTypeObject class after its been + // initialized + new( pymeta ) PyStructMeta; - //this would be the Struct class on python side, it doesnt create any metastruct for itself - if( pymeta -> ht_type.tp_base == &PyStruct::PyType ) - return ( PyObject * ) pymeta; + // this would be the Struct class on python side, it doesnt create any metastruct for itself + if( pymeta->ht_type.tp_base == &PyStruct::PyType ) + return (PyObject *)pymeta; std::string name = PyUnicode_AsUTF8( pyname ); @@ -117,58 +122,88 @@ static PyObject * PyStructMeta_new( PyTypeObject *subtype, PyObject *args, PyObj CSP_THROW( PythonPassthrough, "" ); if( !PyType_Check( type ) && !PyList_Check( type ) ) - CSP_THROW( TypeError, "Struct metadata for key " << keystr << " expected a type, got " << PyObjectPtr::incref( type ) ); + CSP_THROW( TypeError, + "Struct metadata for key " << keystr << " expected a type, got " + << PyObjectPtr::incref( type ) ); std::shared_ptr field; CspTypePtr csptype = pyTypeAsCspType( type ); - switch( csptype -> type() ) + switch( csptype->type() ) { - case csp::CspType::Type::BOOL: field = std::make_shared( keystr ); break; - case csp::CspType::Type::INT64: field = std::make_shared( keystr ); break; - case csp::CspType::Type::DOUBLE: field = std::make_shared( keystr ); break; - case csp::CspType::Type::DATETIME: field = std::make_shared( keystr ); break; - case csp::CspType::Type::TIMEDELTA: field = std::make_shared( keystr ); break; - case csp::CspType::Type::DATE: field = std::make_shared( keystr ); break; - case csp::CspType::Type::TIME: field = std::make_shared( keystr ); break; - case csp::CspType::Type::STRING: field = std::make_shared( csptype, keystr ); break; - case csp::CspType::Type::ENUM: field = std::make_shared( csptype, keystr ); break; - case csp::CspType::Type::STRUCT: field = std::make_shared( csptype, keystr ); break; + case csp::CspType::Type::BOOL: + field = std::make_shared( keystr ); + break; + case csp::CspType::Type::INT64: + field = std::make_shared( keystr ); + break; + case csp::CspType::Type::DOUBLE: + field = std::make_shared( keystr ); + break; + case csp::CspType::Type::DATETIME: + field = std::make_shared( keystr ); + break; + case csp::CspType::Type::TIMEDELTA: + field = std::make_shared( keystr ); + break; + case csp::CspType::Type::DATE: + field = std::make_shared( keystr ); + break; + case csp::CspType::Type::TIME: + field = std::make_shared( keystr ); + break; + case csp::CspType::Type::STRING: + field = std::make_shared( csptype, keystr ); + break; + case csp::CspType::Type::ENUM: + field = std::make_shared( csptype, keystr ); + break; + case csp::CspType::Type::STRUCT: + field = std::make_shared( csptype, keystr ); + break; case csp::CspType::Type::ARRAY: { - const CspArrayType & arrayType = static_cast( *csptype ); - field = ArraySubTypeSwitch::invoke( arrayType.elemType(), [csptype,keystr]( auto tag ) -> std::shared_ptr - { - using CElemType = typename decltype(tag)::type; - using CType = typename CspType::Type::toCArrayType::type; - return std::make_shared>( csptype, keystr ); - } ); + const CspArrayType & arrayType = static_cast( *csptype ); + field = ArraySubTypeSwitch::invoke( + arrayType.elemType(), + [csptype, keystr]( auto tag ) -> std::shared_ptr + { + using CElemType = typename decltype( tag )::type; + using CType = typename CspType::Type::toCArrayType::type; + return std::make_shared>( csptype, keystr ); + } ); break; } - case csp::CspType::Type::DIALECT_GENERIC: field = std::make_shared( keystr, PyTypeObjectPtr::incref( ( PyTypeObject * ) type ) ); break; + case csp::CspType::Type::DIALECT_GENERIC: + field = std::make_shared( keystr, + PyTypeObjectPtr::incref( (PyTypeObject *)type ) ); + break; default: - CSP_THROW( ValueError, "Unexpected csp type " << csptype -> type() << " on struct " << name ); + CSP_THROW( ValueError, "Unexpected csp type " << csptype->type() << " on struct " << name ); } fields.emplace_back( field ); } } std::shared_ptr metabase = nullptr; - Py_ssize_t numbases = PyTuple_GET_SIZE( bases ); + Py_ssize_t numbases = PyTuple_GET_SIZE( bases ); for( Py_ssize_t idx = 0; idx < numbases; ++idx ) { - PyTypeObject * base = ( PyTypeObject * ) PyTuple_GET_ITEM( bases, idx ); + PyTypeObject * base = (PyTypeObject *)PyTuple_GET_ITEM( bases, idx ); - //Check if base is a PyStruct type. Note that this might be the python side csp.Struct class, but we dont create structMeta on that - //( see above ) so it will be null here, no need to check explicitly + // Check if base is a PyStruct type. Note that this might be the python side csp.Struct class, but we dont + // create structMeta on that ( see above ) so it will be null here, no need to check explicitly if( PyType_IsSubtype( base, &PyStruct::PyType ) ) { if( metabase ) - CSP_THROW( TypeError, "Struct " << name << " defined with multiple struct bases. Only single-struct hierarchy is supported" ); - metabase = ( ( PyStructMeta * ) base ) -> structMeta; + CSP_THROW( + TypeError, + "Struct " << name + << " defined with multiple struct bases. Only single-struct hierarchy is supported" ); + metabase = ( (PyStructMeta *)base )->structMeta; } } @@ -188,97 +223,99 @@ static PyObject * PyStructMeta_new( PyTypeObject *subtype, PyObject *args, PyObj | | PyStruct -------------------------- */ - auto structMeta = std::make_shared( ( PyTypeObject * ) pymeta, name, fields, metabase ); + auto structMeta = std::make_shared( (PyTypeObject *)pymeta, name, fields, metabase ); - //Setup fast attr dict lookup - pymeta -> attrDict = PyObjectPtr::own( PyDict_New() ); - for( auto & field : structMeta -> fields() ) + // Setup fast attr dict lookup + pymeta->attrDict = PyObjectPtr::own( PyDict_New() ); + for( auto & field : structMeta->fields() ) { - if( PyDict_SetItem( pymeta -> attrDict.get(), - PyObjectPtr::own( PyUnicode_InternFromString( field -> fieldname().c_str() ) ).get(), - PyObjectPtr::own( PyCapsule_New( field.get(), nullptr, nullptr ) ).get() ) < 0 ) + if( PyDict_SetItem( pymeta->attrDict.get(), + PyObjectPtr::own( PyUnicode_InternFromString( field->fieldname().c_str() ) ).get(), + PyObjectPtr::own( PyCapsule_New( field.get(), nullptr, nullptr ) ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); } - //Setup default image + // Setup default image PyObject * defaults = PyDict_GetItemString( dict, "__defaults__" ); if( !defaults ) - CSP_THROW( KeyError, "StructMeta missing __defaults__" ); \ + CSP_THROW( KeyError, "StructMeta missing __defaults__" ); - //only setup default struct if defaults are defined, otherwise we will do the fast 0-init only + // only setup default struct if defaults are defined, otherwise we will do the fast 0-init only if( PyDict_Size( defaults ) > 0 ) { - StructPtr defaultStruct = structMeta -> create(); + StructPtr defaultStruct = structMeta->create(); PyObject *key, *value; Py_ssize_t pos = 0; while( PyDict_Next( defaults, &pos, &key, &value ) ) { - //Ensure key is properly interned + // Ensure key is properly interned Py_INCREF( key ); PyUnicode_InternInPlace( &key ); PyStruct::setattr( defaultStruct.get(), key, value ); Py_DECREF( key ); } - structMeta -> setDefault( defaultStruct ); + structMeta->setDefault( defaultStruct ); } - pymeta -> structMeta = structMeta; + pymeta->structMeta = structMeta; /*printf( "Struct %s size %lu ( partial %lu )\n", name.c_str(), structMeta -> size(), structMeta -> partialSize() ); for( auto & field : structMeta -> fields() ) { - printf( "\t%s loc %lu size %lu maskloc %lu maskbit %d type %s\n", field -> fieldname().c_str(), field -> offset(), field -> size(), field -> maskOffset(), field -> maskBit(), - field -> type().asCString() ); + printf( "\t%s loc %lu size %lu maskloc %lu maskbit %d type %s\n", field -> fieldname().c_str(), field -> + offset(), field -> size(), field -> maskOffset(), field -> maskBit(), field -> type().asCString() ); }*/ - return ( PyObject * ) pymeta; + return (PyObject *)pymeta; CSP_RETURN_NULL; } void PyStructMeta_dealloc( PyStructMeta * m ) { - CspTypeFactory::instance().removeCachedType( reinterpret_cast( m ) ); - m -> ~PyStructMeta(); - Py_TYPE( m ) -> tp_free( m ); + CspTypeFactory::instance().removeCachedType( reinterpret_cast( m ) ); + m->~PyStructMeta(); + Py_TYPE( m )->tp_free( m ); } PyObject * PyStructMeta_layout( PyStructMeta * m ) { CSP_BEGIN_METHOD; - return PyUnicode_FromString( m -> structMeta -> layout().c_str() ); + return PyUnicode_FromString( m->structMeta->layout().c_str() ); CSP_RETURN_NULL; } static PyObjectPtr PyStructMeta_typeinfo( const CspType * type ) { PyObjectPtr out = PyObjectPtr::own( PyDict_New() ); - if( PyDict_SetItemString( out.get(), "type", PyObjectPtr::own( toPython( type -> type().asString() ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "type", PyObjectPtr::own( toPython( type->type().asString() ) ).get() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "type_id", PyObjectPtr::own( toPython( type -> type().value() ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "type_id", PyObjectPtr::own( toPython( type->type().value() ) ).get() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( type -> type() == CspType::Type::ENUM ) + if( type->type() == CspType::Type::ENUM ) { - auto const * enumType = static_cast( type ); - auto const * enumMeta = static_cast( enumType -> meta().get() ); - if( PyDict_SetItemString( out.get(), "pytype", ( PyObject * ) enumMeta -> pyType().get() ) < 0 ) + auto const * enumType = static_cast( type ); + auto const * enumMeta = static_cast( enumType->meta().get() ); + if( PyDict_SetItemString( out.get(), "pytype", (PyObject *)enumMeta->pyType().get() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); } - else if( type -> type() == CspType::Type::STRUCT ) + else if( type->type() == CspType::Type::STRUCT ) { - auto const * structType = static_cast( type ); - auto const * structMeta = static_cast( structType -> meta().get() ); - if( PyDict_SetItemString( out.get(), "pytype", ( PyObject * ) structMeta -> pyType() ) < 0 ) + auto const * structType = static_cast( type ); + auto const * structMeta = static_cast( structType->meta().get() ); + if( PyDict_SetItemString( out.get(), "pytype", (PyObject *)structMeta->pyType() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); } - else if( type -> type() == CspType::Type::ARRAY ) + else if( type->type() == CspType::Type::ARRAY ) { - auto const * arrayType = static_cast( type ); - if( PyDict_SetItemString( out.get(), "elemtype", PyStructMeta_typeinfo( arrayType -> elemType().get() ).get() ) < 0 ) + auto const * arrayType = static_cast( type ); + if( PyDict_SetItemString( out.get(), "elemtype", PyStructMeta_typeinfo( arrayType->elemType().get() ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); } @@ -288,28 +325,37 @@ static PyObjectPtr PyStructMeta_typeinfo( const CspType * type ) static PyObjectPtr PyStructMeta_fieldinfo( StructField * field ) { PyObjectPtr out = PyObjectPtr::own( PyDict_New() ); - if( PyDict_SetItemString( out.get(), "fieldname", PyObjectPtr::own( toPython( field -> fieldname() ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "fieldname", PyObjectPtr::own( toPython( field->fieldname() ) ).get() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "type", PyStructMeta_typeinfo( field -> type().get() ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "type", PyStructMeta_typeinfo( field->type().get() ).get() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "offset", PyObjectPtr::own( toPython( static_cast( field -> offset() ) ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "offset", + PyObjectPtr::own( toPython( static_cast( field->offset() ) ) ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "size", PyObjectPtr::own( toPython( static_cast( field -> size() ) ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "size", + PyObjectPtr::own( toPython( static_cast( field->size() ) ) ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "alignment", PyObjectPtr::own( toPython( static_cast( field -> alignment() ) ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "alignment", + PyObjectPtr::own( toPython( static_cast( field->alignment() ) ) ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "mask_offset", PyObjectPtr::own( toPython( static_cast( field -> maskOffset() ) ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "mask_offset", + PyObjectPtr::own( toPython( static_cast( field->maskOffset() ) ) ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "mask_bit", PyObjectPtr::own( toPython( field -> maskBit() ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "mask_bit", PyObjectPtr::own( toPython( field->maskBit() ) ).get() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "mask_bitmask", PyObjectPtr::own( toPython( field -> maskBitMask() ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "mask_bitmask", PyObjectPtr::own( toPython( field->maskBitMask() ) ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); return out; @@ -319,157 +365,168 @@ static PyObject * PyStructMeta_metadata_info( PyStructMeta * m ) { PyObjectPtr out = PyObjectPtr::own( PyDict_New() ); - auto * meta = m -> structMeta.get(); + auto * meta = m->structMeta.get(); - PyObjectPtr fields = PyObjectPtr::own( PyList_New( meta -> fields().size() ) ); - for( size_t i = 0; i < meta -> fields().size(); ++i ) - PyList_SET_ITEM( fields.get(), i, PyStructMeta_fieldinfo( meta -> fields()[i].get() ).release() ); + PyObjectPtr fields = PyObjectPtr::own( PyList_New( meta->fields().size() ) ); + for( size_t i = 0; i < meta->fields().size(); ++i ) + PyList_SET_ITEM( fields.get(), i, PyStructMeta_fieldinfo( meta->fields()[i].get() ).release() ); if( PyDict_SetItemString( out.get(), "fields", fields.get() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "size", PyObjectPtr::own( toPython( static_cast( meta -> size() ) ) ).get() )< 0 ) + if( PyDict_SetItemString( out.get(), "size", + PyObjectPtr::own( toPython( static_cast( meta->size() ) ) ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "partial_size", PyObjectPtr::own( toPython( static_cast( meta -> partialSize() ) ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "partial_size", + PyObjectPtr::own( toPython( static_cast( meta->partialSize() ) ) ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "is_native", PyObjectPtr::own( toPython( meta -> isNative() ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "is_native", PyObjectPtr::own( toPython( meta->isNative() ) ).get() ) < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "mask_loc", PyObjectPtr::own( toPython( static_cast( meta -> maskLoc() ) ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "mask_loc", + PyObjectPtr::own( toPython( static_cast( meta->maskLoc() ) ) ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); - if( PyDict_SetItemString( out.get(), "mask_size", PyObjectPtr::own( toPython( static_cast( meta -> maskSize() ) ) ).get() ) < 0 ) + if( PyDict_SetItemString( out.get(), "mask_size", + PyObjectPtr::own( toPython( static_cast( meta->maskSize() ) ) ).get() ) + < 0 ) CSP_THROW( PythonPassthrough, "" ); return out.release(); } -static PyMethodDef PyStructMeta_methods[] = { - {"_layout", (PyCFunction) PyStructMeta_layout, METH_NOARGS, "debug view of structs internal mem layout"}, - {"_metadata_info", (PyCFunction) PyStructMeta_metadata_info, METH_NOARGS, "provide detailed information about struct layout"}, - {NULL} -}; +static PyMethodDef PyStructMeta_methods[] + = { { "_layout", (PyCFunction)PyStructMeta_layout, METH_NOARGS, "debug view of structs internal mem layout" }, + { "_metadata_info", (PyCFunction)PyStructMeta_metadata_info, METH_NOARGS, + "provide detailed information about struct layout" }, + { NULL } }; PyTypeObject PyStructMeta::PyType = { - PyVarObject_HEAD_INIT(nullptr, 0) - "_cspimpl.PyStructMeta", /* tp_name */ - sizeof(PyStructMeta), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) PyStructMeta_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS, /* tp_flags */ - "csp struct metaclass", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyStructMeta_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyType_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /*tp_init*/ - 0,//PyType_GenericAlloc, /* tp_alloc */ - (newfunc) PyStructMeta_new,/* tp_new */ - PyObject_GC_Del, /* tp_free */ + PyVarObject_HEAD_INIT( nullptr, 0 ) "_cspimpl.PyStructMeta", /* tp_name */ + sizeof( PyStructMeta ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyStructMeta_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS, /* tp_flags */ + "csp struct metaclass", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyStructMeta_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyType_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /*tp_init*/ + 0, // PyType_GenericAlloc, /* tp_alloc */ + (newfunc)PyStructMeta_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ }; - -//PyStruct +// PyStruct PyObject * getattr_( const StructField * field, const Struct * struct_ ) { - assert( field -> type() -> type() != CspType::Type::ARRAY ); + assert( field->type()->type() != CspType::Type::ARRAY ); - PyObject *v = switchCspType( field -> type(), [ field, struct_ ]( auto tag ) - { - using CType = typename decltype(tag)::type; - auto *typedField = static_cast::type *>( field ); - return toPython( typedField -> value( struct_ ), *field -> type() ); - } ); + PyObject * v = switchCspType( field->type(), + [field, struct_]( auto tag ) + { + using CType = typename decltype( tag )::type; + auto * typedField + = static_cast::type *>( field ); + return toPython( typedField->value( struct_ ), *field->type() ); + } ); return v; } PyObject * getarrayattr_( const StructField * field, const PyStruct * pystruct ) { - assert( field -> type() -> type() == CspType::Type::ARRAY ); - - const CspArrayType * arrayType = static_cast( field -> type().get() ); - PyObject *v = ArraySubTypeSwitch::invoke( arrayType -> elemType(), [ field, pystruct ]( auto tag ) - { - using StorageT = typename CspType::Type::toCArrayStorageType::type; - using ArrayT = typename StructField::upcast>::type; - auto * typedField = static_cast( field ); - return toPython( typedField -> value( pystruct -> struct_.get() ), *field -> type(), pystruct ); - } ); + assert( field->type()->type() == CspType::Type::ARRAY ); + + const CspArrayType * arrayType = static_cast( field->type().get() ); + PyObject * v = ArraySubTypeSwitch::invoke( + arrayType->elemType(), + [field, pystruct]( auto tag ) + { + using StorageT = typename CspType::Type::toCArrayStorageType::type; + using ArrayT = typename StructField::upcast>::type; + auto * typedField = static_cast( field ); + return toPython( typedField->value( pystruct->struct_.get() ), *field->type(), pystruct ); + } ); return v; } PyObject * PyStruct::getattr( PyObject * attr ) { - auto * field = structMeta() -> field( attr ); + auto * field = structMeta()->field( attr ); if( !field ) return PyObject_GenericGetAttr( this, attr ); - if( !field -> isSet( struct_.get() ) ) + if( !field->isSet( struct_.get() ) ) { - //For efficiency reasons we set err here rather than rely on exceptions, since this - //can get called pretty regularly, ie getattr( s, "f", default ) or hasattr checks - //we also pass the attribute as the exception for efficiency, expensive to format a nice error here - //that wont get used for hasattr calls + // For efficiency reasons we set err here rather than rely on exceptions, since this + // can get called pretty regularly, ie getattr( s, "f", default ) or hasattr checks + // we also pass the attribute as the exception for efficiency, expensive to format a nice error here + // that wont get used for hasattr calls PyErr_SetObject( PyExc_AttributeError, attr ); return nullptr; } - if( field -> type() -> type() == CspType::Type::ARRAY ) + if( field->type()->type() == CspType::Type::ARRAY ) return getarrayattr_( field, this ); - return getattr_( field, ( const Struct * ) struct_.get() ); + return getattr_( field, (const Struct *)struct_.get() ); } void PyStruct::setattr( Struct * s, PyObject * attr, PyObject * value ) { - auto * field = static_cast( s -> meta() ) -> field( attr ); + auto * field = static_cast( s->meta() )->field( attr ); if( !field ) - CSP_THROW( AttributeError, "'" << s -> meta() -> name() << "' object has no attribute '" << PyUnicode_AsUTF8( attr ) << "'" ); + CSP_THROW( AttributeError, + "'" << s->meta()->name() << "' object has no attribute '" << PyUnicode_AsUTF8( attr ) << "'" ); try { - switchCspType( field -> type(), [field,&struct_=s,value]( auto tag ) - { - //workaround for MS compiler bug, separate into two using lines... :/ - using TagType = decltype( tag ); - using CType = typename TagType::type; - using fieldType = typename StructField::upcast::type; - auto *typedField = static_cast( field ); - - if( value ) - typedField -> setValue( struct_, fromPython( value, *field -> type() ) ); - else - typedField -> clearValue( struct_ ); - } ); + switchCspType( field->type(), + [field, &struct_ = s, value]( auto tag ) + { + // workaround for MS compiler bug, separate into two using lines... :/ + using TagType = decltype( tag ); + using CType = typename TagType::type; + using fieldType = typename StructField::upcast::type; + auto * typedField = static_cast( field ); + + if( value ) + typedField->setValue( struct_, fromPython( value, *field->type() ) ); + else + typedField->clearValue( struct_ ); + } ); } catch( const TypeError & err ) { @@ -483,86 +540,93 @@ void PyStruct::setattr( Struct * s, PyObject * attr, PyObject * value ) void repr_struct( const Struct * struct_, std::string & tl_repr, bool show_unset ); // helper functions for formatting to Python standard -void format_bool( const bool val, std::string & tl_repr ) { tl_repr += ( ( val ? "True" : "False" ) ); } +void format_bool( const bool val, std::string & tl_repr ) { tl_repr += ( ( val ? "True" : "False" ) ); } void format_double( const double val, std::string & tl_repr ) { - char * data = PyOS_double_to_string( val, 'r', 0, Py_DTSF_ADD_DOT_0, NULL); + char * data = PyOS_double_to_string( val, 'r', 0, Py_DTSF_ADD_DOT_0, NULL ); tl_repr += std::string( data ); PyMem_Free( data ); } void format_pyobject( const PyObjectPtr & pyptr, std::string & tl_repr ) { auto repr = PyObjectPtr::check( PyObject_Repr( pyptr.get() ) ); - tl_repr += ( char * ) PyUnicode_DATA(repr.get() ); + tl_repr += (char *)PyUnicode_DATA( repr.get() ); } // type checkers to remove switches and do everything at compile-time template -struct is_vector { static bool const value = false; }; +struct is_vector +{ + static bool const value = false; +}; template -struct is_vector > { static bool const value = true; }; +struct is_vector> +{ + static bool const value = true; +}; void repr_field( const Struct * struct_, const StructFieldPtr & field, std::string & tl_repr, bool show_unset ) { // Helper function to convert the values contained in a struct to strings - auto type = field -> type() -> type(); + auto type = field->type()->type(); switch( type ) { case CspType::Type::BOOL: - format_bool( field -> value( struct_ ), tl_repr ); + format_bool( field->value( struct_ ), tl_repr ); break; case CspType::Type::INT8: - tl_repr += std::to_string( field -> value( struct_ ) ); + tl_repr += std::to_string( field->value( struct_ ) ); break; case CspType::Type::UINT8: - tl_repr += std::to_string( field -> value( struct_ ) ); + tl_repr += std::to_string( field->value( struct_ ) ); break; case CspType::Type::INT16: - tl_repr += std::to_string( field -> value( struct_ ) ); + tl_repr += std::to_string( field->value( struct_ ) ); break; case CspType::Type::UINT16: - tl_repr += std::to_string( field -> value( struct_ ) ); + tl_repr += std::to_string( field->value( struct_ ) ); break; case CspType::Type::INT32: - tl_repr += std::to_string( field -> value( struct_ ) ); + tl_repr += std::to_string( field->value( struct_ ) ); break; case CspType::Type::UINT32: - tl_repr += std::to_string( field -> value( struct_ ) ); + tl_repr += std::to_string( field->value( struct_ ) ); break; case CspType::Type::INT64: - tl_repr += std::to_string( field -> value( struct_ ) ); + tl_repr += std::to_string( field->value( struct_ ) ); break; case CspType::Type::UINT64: - tl_repr += std::to_string( field -> value( struct_ ) ); + tl_repr += std::to_string( field->value( struct_ ) ); break; case CspType::Type::DOUBLE: - format_double( field -> value( struct_ ), tl_repr ); + format_double( field->value( struct_ ), tl_repr ); break; case CspType::Type::STRING: - tl_repr += field -> value( struct_ ); + tl_repr += field->value( struct_ ); break; case CspType::Type::STRUCT: { - StructStructField* as_sf = static_cast( field.get() ); - const Struct * val = as_sf -> value( struct_ ).get(); + StructStructField * as_sf = static_cast( field.get() ); + const Struct * val = as_sf->value( struct_ ).get(); repr_struct( val, tl_repr, show_unset ); // recursive break; } case CspType::Type::ARRAY: { - auto const * arrayType = static_cast( field -> type().get() ); - const CspType * elemType = arrayType -> elemType().get(); - - switchCspType( elemType, [ field, struct_, &elemType, &tl_repr, show_unset ]( auto tag ) - { - //workaround for MS compiler bug, separate into two using lines... :/ - using TagType = decltype( tag ); - using CElemType = typename TagType::type; - using ArrayType = typename CspType::Type::toCArrayType::type; - const ArrayType & val = field -> value( struct_ ); - repr_array( val, *elemType, tl_repr, show_unset ); - } ); + auto const * arrayType = static_cast( field->type().get() ); + const CspType * elemType = arrayType->elemType().get(); + + switchCspType( elemType, + [field, struct_, &elemType, &tl_repr, show_unset]( auto tag ) + { + // workaround for MS compiler bug, separate into two using lines... :/ + using TagType = decltype( tag ); + using CElemType = typename TagType::type; + using ArrayType = typename CspType::Type::toCArrayType::type; + const ArrayType & val = field->value( struct_ ); + repr_array( val, *elemType, tl_repr, show_unset ); + } ); break; } @@ -592,8 +656,10 @@ void repr_array( const std::vector & val, const CspType & elemType, st bool first = true; for( auto it = val.begin(); it != val.end(); it++ ) { - if( unlikely( first ) ) first = false; - else tl_repr += ", "; + if( unlikely( first ) ) + first = false; + else + tl_repr += ", "; // print array types if constexpr( std::is_same::value ) @@ -627,31 +693,33 @@ void repr_struct( const Struct * struct_, std::string & tl_repr, bool show_unset static thread_local std::unordered_set tl_structReprsVisited; auto [_, inserted] = tl_structReprsVisited.insert( struct_ ); - if ( !inserted ) + if( !inserted ) { tl_repr += "( ... )"; return; } - auto * meta = struct_ -> meta(); + auto * meta = struct_->meta(); // Helper function to convert a struct to a string representation - tl_repr += meta -> name(); + tl_repr += meta->name(); tl_repr += "( "; // Extract fields and attributes bool first = true; - for( auto & fieldname : meta -> fieldNames() ) + for( auto & fieldname : meta->fieldNames() ) { - auto & field = meta -> field( fieldname ); - if( !( field -> isSet( struct_ ) ) && !show_unset ) + auto & field = meta->field( fieldname ); + if( !( field->isSet( struct_ ) ) && !show_unset ) continue; - if( unlikely( first ) ) first = false; - else tl_repr += ", "; + if( unlikely( first ) ) + first = false; + else + tl_repr += ", "; tl_repr += fieldname; tl_repr += "="; - if( !( field -> isSet( struct_ ) ) ) + if( !( field->isSet( struct_ ) ) ) tl_repr += ""; else repr_field( struct_, field, tl_repr, show_unset ); @@ -666,8 +734,8 @@ PyObject * PyStruct::repr( bool show_unset ) const static thread_local std::string tl_repr; // Each struct is responsible for clearing the TLS string after - size_t offset = tl_repr.size(); - const Struct * val = this -> struct_.get(); + size_t offset = tl_repr.size(); + const Struct * val = this->struct_.get(); repr_struct( val, tl_repr, show_unset ); PyObject * rv = PyUnicode_FromString( tl_repr.c_str() + offset ); @@ -675,37 +743,36 @@ PyObject * PyStruct::repr( bool show_unset ) const return rv; } - /*************************** A note on GC madness... -We ensure to maintain a single PyStruct instance per c++ StructPtr instance. This instance is cached in the Struct's hidden() data and will -be created on the fly if it doesnt exist when needed ( see Conversions.h ) -The instance on the StructPtr does NOT own a python ref. When the PyStruct refcount drops to 0, or when its garbage collected and destroyed, -we clear the dialectPtr from the c++ struct. -Now in tp_traverse and tp_clear, we ensure not to do traversal or clearing unless we are the only holders of the c++ StructPtr, otherwise we -would end up clearing / freeing data that is still used by the struct ( instance can still exist in c++ / in the timeseries buffers ) -The reason we ensure a single PyStruct instance exists is to avoid leaks where we can have a self referencing container with the same struct -twice, ie [ S1, S2 ] where S1 and S2 point to the same StructPtr and hold a ref to the list as well. If we allowed multiple PyStruct instances, the -refcount would stay at 2 and this would never clean up +We ensure to maintain a single PyStruct instance per c++ StructPtr instance. This instance is cached in the Struct's +hidden() data and will be created on the fly if it doesnt exist when needed ( see Conversions.h ) The instance on the +StructPtr does NOT own a python ref. When the PyStruct refcount drops to 0, or when its garbage collected and +destroyed, we clear the dialectPtr from the c++ struct. Now in tp_traverse and tp_clear, we ensure not to do traversal +or clearing unless we are the only holders of the c++ StructPtr, otherwise we would end up clearing / freeing data that +is still used by the struct ( instance can still exist in c++ / in the timeseries buffers ) The reason we ensure a +single PyStruct instance exists is to avoid leaks where we can have a self referencing container with the same struct +twice, ie [ S1, S2 ] where S1 and S2 point to the same StructPtr and hold a ref to the list as well. If we allowed +multiple PyStruct instances, the refcount would stay at 2 and this would never clean up ****************************/ -PyObject * PyStruct_new( PyTypeObject * type, PyObject *args, PyObject *kwds ) +PyObject * PyStruct_new( PyTypeObject * type, PyObject * args, PyObject * kwds ) { CSP_BEGIN_METHOD; - //base struct class - if( ! ( (PyStructMeta * ) type ) -> structMeta ) + // base struct class + if( !( (PyStructMeta *)type )->structMeta ) CSP_THROW( TypeError, "csp.Struct cannot be instantiated" ); - StructPtr struct_ = ( (PyStructMeta * ) type ) -> structMeta -> create(); + StructPtr struct_ = ( (PyStructMeta *)type )->structMeta->create(); - PyStruct * pystruct = ( PyStruct * ) type -> tp_alloc( type, 0 ); + PyStruct * pystruct = (PyStruct *)type->tp_alloc( type, 0 ); - //assign dialectptr, but we DO NOT incref the instance on the struct - struct_ -> setDialectPtr( pystruct ); + // assign dialectptr, but we DO NOT incref the instance on the struct + struct_->setDialectPtr( pystruct ); - new ( pystruct ) PyStruct( std::move( struct_ ) ); + new( pystruct ) PyStruct( std::move( struct_ ) ); return pystruct; CSP_RETURN_NULL; @@ -713,18 +780,18 @@ PyObject * PyStruct_new( PyTypeObject * type, PyObject *args, PyObject *kwds ) int PyStruct_tp_clear( PyStruct * self ) { - //important to only invoke cleanup if we are the actual last refcount holder - if( self -> struct_ -> refcount() != 1 ) + // important to only invoke cleanup if we are the actual last refcount holder + if( self->struct_->refcount() != 1 ) return 0; - for( auto & field : self -> structMeta() -> fields() ) + for( auto & field : self->structMeta()->fields() ) { - if( field -> type() -> type() == CspType::Type::DIALECT_GENERIC && field -> isSet( self -> struct_.get() ) ) + if( field->type()->type() == CspType::Type::DIALECT_GENERIC && field->isSet( self->struct_.get() ) ) { - auto * pyfield = static_cast( field.get() ); - PyObject * o = reinterpret_cast(pyfield -> value( self -> struct_.get() ) ).ptr(); + auto * pyfield = static_cast( field.get() ); + PyObject * o = reinterpret_cast( pyfield->value( self->struct_.get() ) ).ptr(); Py_XINCREF( o ); - pyfield -> clearValue( self -> struct_.get() ); + pyfield->clearValue( self->struct_.get() ); Py_CLEAR( o ); } } @@ -734,16 +801,16 @@ int PyStruct_tp_clear( PyStruct * self ) int PyStruct_traverse( PyStruct * self, visitproc visit, void * arg ) { - //important to only traverse the actual last refcount holder - if( self -> struct_ -> refcount() != 1 ) + // important to only traverse the actual last refcount holder + if( self->struct_->refcount() != 1 ) return 0; - for( auto & field : self -> structMeta() -> fields() ) + for( auto & field : self->structMeta()->fields() ) { - if( field -> type() -> type() == CspType::Type::DIALECT_GENERIC ) + if( field->type()->type() == CspType::Type::DIALECT_GENERIC ) { - auto * pyfield = static_cast( field.get() ); - Py_VISIT( reinterpret_cast( pyfield -> value( self -> struct_.get() ) ).ptr() ); + auto * pyfield = static_cast( field.get() ); + Py_VISIT( reinterpret_cast( pyfield->value( self->struct_.get() ) ).ptr() ); } } @@ -755,17 +822,19 @@ void PyStruct_dealloc( PyStruct * self ) PyObject_GC_UnTrack( self ); PyStruct_tp_clear( self ); - //clear the dialectPtr at this point - self -> struct_ -> setDialectPtr( nullptr ); + // clear the dialectPtr at this point + self->struct_->setDialectPtr( nullptr ); - self -> ~PyStruct(); - Py_TYPE( self ) -> tp_free( self ); + self->~PyStruct(); + Py_TYPE( self )->tp_free( self ); } void PyStruct_setattrs( PyStruct * self, PyObject * args, PyObject * kwargs, const char * methodName ) { if( PyTuple_GET_SIZE( args ) > 0 ) - CSP_THROW( TypeError, "'" << self -> ob_type -> tp_name << '.' << methodName << "' takes 0 positional arguments but " << PyTuple_GET_SIZE( args ) << " were given" ); + CSP_THROW( TypeError, + "'" << self->ob_type->tp_name << '.' << methodName << "' takes 0 positional arguments but " + << PyTuple_GET_SIZE( args ) << " were given" ); if( !kwargs ) return; @@ -776,7 +845,7 @@ void PyStruct_setattrs( PyStruct * self, PyObject * args, PyObject * kwargs, con { Py_INCREF( key ); PyUnicode_InternInPlace( &key ); - self -> setattr( key, value ); + self->setattr( key, value ); Py_DECREF( key ); } } @@ -803,17 +872,16 @@ PyObject * PyStruct_richcompare( PyStruct * self, PyObject * other, int op ) { CSP_BEGIN_METHOD; - if( !PyType_IsSubtype( other -> ob_type, &PyStruct::PyType ) || - ( op != Py_EQ && op != Py_NE ) ) + if( !PyType_IsSubtype( other->ob_type, &PyStruct::PyType ) || ( op != Py_EQ && op != Py_NE ) ) { Py_INCREF( Py_NotImplemented ); return Py_NotImplemented; } - PyStruct * rhs = ( PyStruct * ) other; + PyStruct * rhs = (PyStruct *)other; - bool rv = ( *self -> struct_.get() ) == ( *rhs -> struct_.get() ); - rv = ( rv == ( op == Py_EQ ) ); + bool rv = ( *self->struct_.get() ) == ( *rhs->struct_.get() ); + rv = ( rv == ( op == Py_EQ ) ); return toPython( rv ); CSP_RETURN_NONE; @@ -823,8 +891,8 @@ Py_hash_t PyStruct_hash( PyStruct * self ) { CSP_BEGIN_METHOD; - Py_hash_t hash = self -> struct_ -> hash(); - //wheres the "so incredibly unlikely" macro + Py_hash_t hash = self->struct_->hash(); + // wheres the "so incredibly unlikely" macro if( unlikely( hash == -1 ) ) hash = 2; @@ -837,7 +905,7 @@ PyObject * PyStruct_str( PyStruct * self ) { CSP_BEGIN_METHOD; - return self -> repr( true ); + return self->repr( true ); CSP_RETURN_NULL; } @@ -846,17 +914,16 @@ PyObject * PyStruct_repr( PyStruct * self ) { CSP_BEGIN_METHOD; - return self -> repr( false ); + return self->repr( false ); CSP_RETURN_NULL; } - PyObject * PyStruct_getattro( PyStruct * self, PyObject * attr ) { CSP_BEGIN_METHOD; - return self -> getattr( attr ); + return self->getattr( attr ); CSP_RETURN_NULL; } @@ -865,7 +932,7 @@ int PyStruct_setattro( PyStruct * self, PyObject * attr, PyObject * value ) { CSP_BEGIN_METHOD; - self -> setattr( attr, value ); + self->setattr( attr, value ); CSP_RETURN_INT; } @@ -873,8 +940,8 @@ int PyStruct_setattro( PyStruct * self, PyObject * attr, PyObject * value ) PyObject * PyStruct_copy( PyStruct * self ) { CSP_BEGIN_METHOD; - PyObject * copy = self -> ob_type -> tp_alloc( self -> ob_type, 0 ); - new ( copy ) PyStruct( self -> struct_ -> copy() ); + PyObject * copy = self->ob_type->tp_alloc( self->ob_type, 0 ); + new( copy ) PyStruct( self->struct_->copy() ); return copy; CSP_RETURN_NULL; } @@ -882,12 +949,12 @@ PyObject * PyStruct_copy( PyStruct * self ) PyObject * PyStruct_deepcopy( PyStruct * self ) { CSP_BEGIN_METHOD; - //Note that once tp_alloc is called, the object will get added to GC - //deepcopy traversal may kick in a GC collect, so we have to call that first before the PyStruct is created - //of it may traverse a partially consturcted object and crash - auto deepcopy = self -> struct_ -> deepcopy(); - PyObject * pyDeepcopy = self -> ob_type -> tp_alloc( self -> ob_type, 0 ); - new ( pyDeepcopy ) PyStruct( deepcopy ); + // Note that once tp_alloc is called, the object will get added to GC + // deepcopy traversal may kick in a GC collect, so we have to call that first before the PyStruct is created + // of it may traverse a partially consturcted object and crash + auto deepcopy = self->struct_->deepcopy(); + PyObject * pyDeepcopy = self->ob_type->tp_alloc( self->ob_type, 0 ); + new( pyDeepcopy ) PyStruct( deepcopy ); return pyDeepcopy; CSP_RETURN_NULL; } @@ -895,7 +962,7 @@ PyObject * PyStruct_deepcopy( PyStruct * self ) PyObject * PyStruct_clear( PyStruct * self ) { CSP_BEGIN_METHOD; - self -> struct_ -> clear(); + self->struct_->clear(); CSP_RETURN_NONE; } @@ -904,9 +971,9 @@ PyObject * PyStruct_copy_from( PyStruct * self, PyObject * o ) CSP_BEGIN_METHOD; if( !PyType_IsSubtype( Py_TYPE( o ), &PyStruct::PyType ) ) - CSP_THROW( TypeError, "Attempting to copy from non-struct type " << Py_TYPE( o ) -> tp_name ); + CSP_THROW( TypeError, "Attempting to copy from non-struct type " << Py_TYPE( o )->tp_name ); - self -> struct_ -> copyFrom( ( ( PyStruct * ) o ) -> struct_.get() ); + self->struct_->copyFrom( ( (PyStruct *)o )->struct_.get() ); CSP_RETURN_NONE; } @@ -915,9 +982,9 @@ PyObject * PyStruct_deepcopy_from( PyStruct * self, PyObject * o ) CSP_BEGIN_METHOD; if( !PyType_IsSubtype( Py_TYPE( o ), &PyStruct::PyType ) ) - CSP_THROW( TypeError, "Attempting to deepcopy from non-struct type " << Py_TYPE( o ) -> tp_name ); + CSP_THROW( TypeError, "Attempting to deepcopy from non-struct type " << Py_TYPE( o )->tp_name ); - self -> struct_ -> deepcopyFrom( ( ( PyStruct * ) o ) -> struct_.get() ); + self->struct_->deepcopyFrom( ( (PyStruct *)o )->struct_.get() ); CSP_RETURN_NONE; } @@ -926,16 +993,13 @@ PyObject * PyStruct_update_from( PyStruct * self, PyObject * o ) CSP_BEGIN_METHOD; if( !PyType_IsSubtype( Py_TYPE( o ), &PyStruct::PyType ) ) - CSP_THROW( TypeError, "Attempting to update_from from non-struct type " << Py_TYPE( o ) -> tp_name ); + CSP_THROW( TypeError, "Attempting to update_from from non-struct type " << Py_TYPE( o )->tp_name ); - self -> struct_ -> updateFrom( ( ( PyStruct * ) o ) -> struct_.get() ); + self->struct_->updateFrom( ( (PyStruct *)o )->struct_.get() ); CSP_RETURN_NONE; } -PyObject * PyStruct_all_fields_set( PyStruct * self ) -{ - return toPython( self -> struct_ -> allFieldsSet() ); -} +PyObject * PyStruct_all_fields_set( PyStruct * self ) { return toPython( self->struct_->allFieldsSet() ); } PyObject * PyStruct_to_dict( PyStruct * self, PyObject * args, PyObject * kwargs ) { @@ -947,13 +1011,14 @@ PyObject * PyStruct_to_dict( PyStruct * self, PyObject * args, PyObject * kwargs { if( ( callable != Py_None ) && !PyCallable_Check( callable ) ) { - CSP_THROW( TypeError, "Parameter must be callable or None got " + std::string( Py_TYPE( callable ) -> tp_name ) ); + CSP_THROW( TypeError, + "Parameter must be callable or None got " + std::string( Py_TYPE( callable )->tp_name ) ); } } if( callable == Py_None ) callable = nullptr; - auto struct_ptr = self -> struct_; - auto pyobj_ptr = structToDict( struct_ptr, callable ); + auto struct_ptr = self->struct_; + auto pyobj_ptr = structToDict( struct_ptr, callable ); return pyobj_ptr.release(); CSP_RETURN_NULL; @@ -977,71 +1042,75 @@ PyObject * PyStruct_to_json( PyStruct * self, PyObject * args, PyObject * kwargs { CSP_THROW( TypeError, "Expected a callable as the argument" ); } - auto struct_ptr = self -> struct_; - auto buffer = structToJson( struct_ptr, callable ); + auto struct_ptr = self->struct_; + auto buffer = structToJson( struct_ptr, callable ); return toPython( buffer ); CSP_RETURN_NULL; } -static PyMethodDef PyStruct_methods[] = { - { "copy", (PyCFunction) PyStruct_copy, METH_NOARGS, "make a shallow copy of the struct" }, - { "deepcopy", (PyCFunction) PyStruct_deepcopy, METH_NOARGS, "make a deep copy of the struct" }, - { "clear", (PyCFunction) PyStruct_clear, METH_NOARGS, "clear all fields" }, - { "copy_from", (PyCFunction) PyStruct_copy_from, METH_O, "copy from struct. struct must be same type or a derived type. unset fields will copy over" }, - { "deepcopy_from", (PyCFunction) PyStruct_deepcopy_from, METH_O, "deepcopy from struct. struct must be same type or a derived type. unset fields will copy over" }, - { "update_from", (PyCFunction) PyStruct_update_from, METH_O, "update from struct. struct must be same type or a derived type. unset fields will be not be copied" }, - { "update", (PyCFunction) PyStruct_update, METH_VARARGS | METH_KEYWORDS, "update from key=val. given fields will be set on struct. other fields will remain as is in struct" }, - { "all_fields_set", (PyCFunction) PyStruct_all_fields_set, METH_NOARGS, "return true if all fields on the struct are set" }, - { "to_dict", (PyCFunction) PyStruct_to_dict, METH_VARARGS | METH_KEYWORDS, "return a python dict of the struct by recursively converting struct members into python dicts" }, - { "to_json", (PyCFunction) PyStruct_to_json, METH_VARARGS | METH_KEYWORDS, "return a json string of the struct by recursively converting struct members into json format" }, - { NULL} -}; +static PyMethodDef PyStruct_methods[] + = { { "copy", (PyCFunction)PyStruct_copy, METH_NOARGS, "make a shallow copy of the struct" }, + { "deepcopy", (PyCFunction)PyStruct_deepcopy, METH_NOARGS, "make a deep copy of the struct" }, + { "clear", (PyCFunction)PyStruct_clear, METH_NOARGS, "clear all fields" }, + { "copy_from", (PyCFunction)PyStruct_copy_from, METH_O, + "copy from struct. struct must be same type or a derived type. unset fields will copy over" }, + { "deepcopy_from", (PyCFunction)PyStruct_deepcopy_from, METH_O, + "deepcopy from struct. struct must be same type or a derived type. unset fields will copy over" }, + { "update_from", (PyCFunction)PyStruct_update_from, METH_O, + "update from struct. struct must be same type or a derived type. unset fields will be not be copied" }, + { "update", (PyCFunction)PyStruct_update, METH_VARARGS | METH_KEYWORDS, + "update from key=val. given fields will be set on struct. other fields will remain as is in struct" }, + { "all_fields_set", (PyCFunction)PyStruct_all_fields_set, METH_NOARGS, + "return true if all fields on the struct are set" }, + { "to_dict", (PyCFunction)PyStruct_to_dict, METH_VARARGS | METH_KEYWORDS, + "return a python dict of the struct by recursively converting struct members into python dicts" }, + { "to_json", (PyCFunction)PyStruct_to_json, METH_VARARGS | METH_KEYWORDS, + "return a json string of the struct by recursively converting struct members into json format" }, + { NULL } }; PyTypeObject PyStruct::PyType = { - PyVarObject_HEAD_INIT(nullptr, 0) - "_cspimpl.PyStruct", /* tp_name */ - sizeof(PyStruct), /* tp_basicsize */ - 0, /* tp_itemsize */ - ( destructor ) PyStruct_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - ( reprfunc ) PyStruct_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - ( hashfunc ) PyStruct_hash, /* tp_hash */ - 0, /* tp_call */ - ( reprfunc ) PyStruct_str, /* tp_str */ - ( getattrofunc ) PyStruct_getattro, /* tp_getattro */ - ( setattrofunc ) PyStruct_setattro, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - "csp struct", /* tp_doc */ - ( traverseproc ) PyStruct_traverse, /* tp_traverse */ - ( inquiry ) PyStruct_tp_clear, /* tp_clear */ - ( richcmpfunc ) PyStruct_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyStruct_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - ( initproc ) PyStruct_init, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - ( newfunc ) PyStruct_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ + PyVarObject_HEAD_INIT( nullptr, 0 ) "_cspimpl.PyStruct", /* tp_name */ + sizeof( PyStruct ), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyStruct_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)PyStruct_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + (hashfunc)PyStruct_hash, /* tp_hash */ + 0, /* tp_call */ + (reprfunc)PyStruct_str, /* tp_str */ + (getattrofunc)PyStruct_getattro, /* tp_getattro */ + (setattrofunc)PyStruct_setattro, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ + "csp struct", /* tp_doc */ + (traverseproc)PyStruct_traverse, /* tp_traverse */ + (inquiry)PyStruct_tp_clear, /* tp_clear */ + (richcmpfunc)PyStruct_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyStruct_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)PyStruct_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + (newfunc)PyStruct_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ }; REGISTER_TYPE_INIT( &PyStructMeta::PyType, "PyStructMeta" ) -REGISTER_TYPE_INIT( &PyStruct::PyType, "PyStruct" ) +REGISTER_TYPE_INIT( &PyStruct::PyType, "PyStruct" ) // Instantiate all templates for PyStructList class template struct PyStructList; @@ -1081,4 +1150,4 @@ template struct PyStructFastList; template struct PyStructFastList; template struct PyStructFastList; -} +} // namespace csp::python diff --git a/cpp/csp/python/PyStructToDict.cpp b/cpp/csp/python/PyStructToDict.cpp index 9f521276..12077e5b 100644 --- a/cpp/csp/python/PyStructToDict.cpp +++ b/cpp/csp/python/PyStructToDict.cpp @@ -1,13 +1,15 @@ -#include #include +#include namespace csp::python { static thread_local std::unordered_set g_tl_ptrsVisited; -class CircularRefCheck { +class CircularRefCheck +{ public: - CircularRefCheck( const void * ptr ): m_ptr( ptr ) + CircularRefCheck( const void * ptr ) + : m_ptr( ptr ) { auto [_, inserted] = g_tl_ptrsVisited.insert( m_ptr ); if( !inserted ) @@ -16,17 +18,14 @@ class CircularRefCheck { } } - ~CircularRefCheck() - { - g_tl_ptrsVisited.erase( m_ptr ); - } + ~CircularRefCheck() { g_tl_ptrsVisited.erase( m_ptr ); } private: const void * m_ptr; }; // Helper function to convert csp Structs into python objects recursively -PyObjectPtr parseStructToDictRecursive( const StructPtr& self, PyObject * callable ); +PyObjectPtr parseStructToDictRecursive( const StructPtr & self, PyObject * callable ); // Helper function to parse some python objects in cpp, this should not be used extensively. // instead add support for those python types to csp so that they can be handled more generically @@ -35,7 +34,7 @@ PyObjectPtr parsePyObject( PyObject * value, PyObject * callable, bool is_recurs // Helper fallback function to convert any type into python object recursively template -inline PyObjectPtr parseCspToPython( const T& val, const CspType& typ, PyObject * callable ) +inline PyObjectPtr parseCspToPython( const T & val, const CspType & typ, PyObject * callable ) { // Default handler for any unknown T return PyObjectPtr::own( toPython( val ) ); @@ -43,7 +42,7 @@ inline PyObjectPtr parseCspToPython( const T& val, const CspType& typ, PyObject // Helper function to convert Enums into python object recursively template<> -inline PyObjectPtr parseCspToPython( const CspEnum& val, const CspType& typ, PyObject * callable ) +inline PyObjectPtr parseCspToPython( const CspEnum & val, const CspType & typ, PyObject * callable ) { // NOTE: Customization parameter to return the enum instead of string to be added return PyObjectPtr::own( toPython( val.name() ) ); @@ -51,14 +50,14 @@ inline PyObjectPtr parseCspToPython( const CspEnum& val, const CspType& typ, PyO // Helper function to convert csp Structs into python object recursively template<> -inline PyObjectPtr parseCspToPython( const StructPtr& val, const CspType& typ, PyObject * callable ) +inline PyObjectPtr parseCspToPython( const StructPtr & val, const CspType & typ, PyObject * callable ) { return parseStructToDictRecursive( val, callable ); } // Helper function to convert python objects in csp Structs into python object recursively template<> -inline PyObjectPtr parseCspToPython( const DialectGenericType& val, const CspType& typ, PyObject * callable ) +inline PyObjectPtr parseCspToPython( const DialectGenericType & val, const CspType & typ, PyObject * callable ) { auto py_obj = PyObjectPtr::own( toPython( val ) ); return parsePyObject( py_obj.get(), callable, false ); @@ -66,13 +65,13 @@ inline PyObjectPtr parseCspToPython( const DialectGenericType& val, const CspTyp // Helper function to convert arrays in csp Structs into python lists recursively template -inline PyObjectPtr parseCspToPython( const std::vector& val, const CspType& typ, PyObject * callable ) +inline PyObjectPtr parseCspToPython( const std::vector & val, const CspType & typ, PyObject * callable ) { using ElemT = typename CspType::Type::toCArrayElemType::type; - auto const * arrayType = static_cast( &typ ); - const CspType * elemType = arrayType -> elemType().get(); - auto new_list = PyObjectPtr::own( PyList_New( val.size() ) ); + auto const * arrayType = static_cast( &typ ); + const CspType * elemType = arrayType->elemType().get(); + auto new_list = PyObjectPtr::own( PyList_New( val.size() ) ); for( size_t idx = 0; idx < val.size(); ++idx ) { @@ -84,37 +83,41 @@ inline PyObjectPtr parseCspToPython( const std::vector& val, const Csp return new_list; } -PyObjectPtr parseStructToDictRecursive( const StructPtr& self, PyObject * callable ) +PyObjectPtr parseStructToDictRecursive( const StructPtr & self, PyObject * callable ) { auto * struct_ptr = self.get(); CircularRefCheck checker( struct_ptr ); - auto * meta = static_cast( self -> meta() ); + auto * meta = static_cast( self->meta() ); auto new_dict = PyObjectPtr::own( PyDict_New() ); - auto& fields = meta -> fields(); + auto & fields = meta->fields(); - for( const auto& field: fields ) + for( const auto & field : fields ) { // NOTE: Add customization parameter to skip fields starting with underscore("_") - if( !field -> isSet( struct_ptr ) ) + if( !field->isSet( struct_ptr ) ) continue; - auto& key = field -> fieldname(); - auto py_obj = switchCspType( field -> type(), [field, struct_ptr, callable]( auto tag ) - { - using CType = typename decltype( tag )::type; - auto * typedField = static_cast::type *>( field.get() ); - return parseCspToPython( typedField -> value( struct_ptr ), *field -> type(), callable ); - } ); + auto & key = field->fieldname(); + auto py_obj + = switchCspType( field->type(), + [field, struct_ptr, callable]( auto tag ) + { + using CType = typename decltype( tag )::type; + auto * typedField + = static_cast::type *>( field.get() ); + return parseCspToPython( typedField->value( struct_ptr ), *field->type(), callable ); + } ); PyDict_SetItemString( new_dict.get(), key.c_str(), py_obj.get() ); } // Optional postprocess hook in python to allow caller to customize to_dict behavior for struct - PyObject * py_type = ( PyObject * ) meta -> pyType(); + PyObject * py_type = (PyObject *)meta->pyType(); if( PyObject_HasAttrString( py_type, "postprocess_to_dict" ) ) { auto postprocess_dict_callable = PyObjectPtr::own( PyObject_GetAttrString( py_type, "postprocess_to_dict" ) ); - new_dict = PyObjectPtr::check( PyObject_CallFunction( postprocess_dict_callable.get(), "(O)", new_dict.get() ) ); + new_dict + = PyObjectPtr::check( PyObject_CallFunction( postprocess_dict_callable.get(), "(O)", new_dict.get() ) ); } return new_dict; } @@ -122,8 +125,9 @@ PyObjectPtr parseStructToDictRecursive( const StructPtr& self, PyObject * callab class PySequenceIterator { public: - PySequenceIterator( PyObject * iter, PyObject * callable ): - m_iter( iter ), m_callable( callable ) + PySequenceIterator( PyObject * iter, PyObject * callable ) + : m_iter( iter ) + , m_callable( callable ) { } PyObject * iternext() @@ -139,6 +143,7 @@ class PySequenceIterator return parsed_obj.release(); } } + private: PyObject * m_iter; PyObject * m_callable; @@ -152,20 +157,20 @@ PyObjectPtr parsePySequence( PyObject * py_seq, PyObject * callable ) if( raw_iter.get() == NULL ) CSP_THROW( ValueError, "Cannot extract iterator from python sequence" ); PySequenceIterator py_seq_iter( raw_iter.get(), callable ); - auto iter = PyObjectPtr::own( PyIterator::create( py_seq_iter ) ); - PyTypeObject * orig_type = py_seq -> ob_type; - return PyObjectPtr::own( PyObject_CallFunction( ( PyObject * ) orig_type, "(O)", iter.get() ) ); + auto iter = PyObjectPtr::own( PyIterator::create( py_seq_iter ) ); + PyTypeObject * orig_type = py_seq->ob_type; + return PyObjectPtr::own( PyObject_CallFunction( (PyObject *)orig_type, "(O)", iter.get() ) ); } // Helper function to parse python dicts recursively PyObjectPtr parsePyDict( PyObject * py_dict, PyObject * callable ) { CircularRefCheck checker( py_dict ); - PyObject * py_key = NULL; - PyObject * py_value = NULL; - Py_ssize_t pos = 0; - PyTypeObject * orig_type = py_dict -> ob_type; - auto parsed_dict = PyObjectPtr::own( PyObject_CallFunction( ( PyObject * ) orig_type, "" ) ); + PyObject * py_key = NULL; + PyObject * py_value = NULL; + Py_ssize_t pos = 0; + PyTypeObject * orig_type = py_dict->ob_type; + auto parsed_dict = PyObjectPtr::own( PyObject_CallFunction( (PyObject *)orig_type, "" ) ); while( PyDict_Next( py_dict, &pos, &py_key, &py_value ) ) { auto py_obj = parsePyObject( py_value, callable, false ); @@ -178,11 +183,11 @@ PyObjectPtr parsePyObject( PyObject * value, PyObject * callable, bool is_recurs { INIT_PYDATETIME; - if( ( value == Py_None ) || // None check - ( PyBool_Check( value ) || PyLong_Check( value ) || PyFloat_Check( value ) ) || // Primitives check - ( PyUnicode_Check( value ) || PyBytes_Check( value ) ) || // Unicode/bytes check - ( PyTime_CheckExact( value ) || PyDate_CheckExact( value ) || - PyDateTime_CheckExact( value ) || PyDelta_CheckExact( value ) ) ) // Datetime check + if( ( value == Py_None ) || // None check + ( PyBool_Check( value ) || PyLong_Check( value ) || PyFloat_Check( value ) ) || // Primitives check + ( PyUnicode_Check( value ) || PyBytes_Check( value ) ) || // Unicode/bytes check + ( PyTime_CheckExact( value ) || PyDate_CheckExact( value ) || PyDateTime_CheckExact( value ) + || PyDelta_CheckExact( value ) ) ) // Datetime check return PyObjectPtr::incref( value ); else if( PyTuple_Check( value ) || PyList_Check( value ) || PySet_Check( value ) ) return parsePySequence( value, callable ); @@ -190,12 +195,12 @@ PyObjectPtr parsePyObject( PyObject * value, PyObject * callable, bool is_recurs return parsePyDict( value, callable ); else if( PyType_IsSubtype( Py_TYPE( value ), &PyStruct::PyType ) ) { - auto struct_ptr = static_cast( value ) -> struct_; + auto struct_ptr = static_cast( value )->struct_; return parseStructToDictRecursive( struct_ptr, callable ); } else if( PyType_IsSubtype( Py_TYPE( value ), &PyCspEnum::PyType ) ) { - auto enum_ptr = static_cast( value ) -> enum_; + auto enum_ptr = static_cast( value )->enum_; return parseCspToPython( enum_ptr, CspType( CspType::Type::ENUM ), callable ); } else @@ -215,11 +220,11 @@ PyObjectPtr parsePyObject( PyObject * value, PyObject * callable, bool is_recurs } } -PyObjectPtr structToDict( const StructPtr& struct_ptr, PyObject * callable ) +PyObjectPtr structToDict( const StructPtr & struct_ptr, PyObject * callable ) { // Reset circular reference checker state g_tl_ptrsVisited.clear(); return parseStructToDictRecursive( struct_ptr, callable ); } -} +} // namespace csp::python diff --git a/cpp/csp/python/PyStructToJson.cpp b/cpp/csp/python/PyStructToJson.cpp index b4b9edc5..20002f6e 100644 --- a/cpp/csp/python/PyStructToJson.cpp +++ b/cpp/csp/python/PyStructToJson.cpp @@ -6,23 +6,23 @@ namespace csp::python { // Helper function to convert csp Structs into json objects recursively -rapidjson::Value toJsonRecursive( const StructPtr& self, rapidjson::Document& doc, PyObject * callable ); +rapidjson::Value toJsonRecursive( const StructPtr & self, rapidjson::Document & doc, PyObject * callable ); // Helper function to parse some python objects in cpp, this should not be used extensively. // instead add support for those python types to csp so that they can be handled more generically // and in a language agnostic way -rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyObject * callable, bool is_recursing ); +rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document & doc, PyObject * callable, bool is_recursing ); // Helper fallback function to convert any type into json format recursively template -inline rapidjson::Value toJson( const T& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const T & val, const CspType & typ, rapidjson::Document & doc, PyObject * callable ) { // Default handler for any unknown T return rapidjson::Value( val ); } // Helper function for parsing doubles -inline rapidjson::Value doubleToJson( const double& val, rapidjson::Document& doc ) +inline rapidjson::Value doubleToJson( const double & val, rapidjson::Document & doc ) { // NOTE: Rapidjson adds support for this in a future release. Remove this when we upgrade rapidjson to a version // after 07/16/2023 and use kWriteNanAndInfNullFlag in the writer. @@ -30,7 +30,7 @@ inline rapidjson::Value doubleToJson( const double& val, rapidjson::Document& do // To be compatible with other JSON libraries, we cannot use the default approach that rapidjson has to // serializing NaN, and (+/-)Infs. We need to manually convert them to NULLs. Rapidjson adds support for this // in a future release. - if ( std::isnan( val ) || std::isinf( val ) ) + if( std::isnan( val ) || std::isinf( val ) ) { return rapidjson::Value(); } @@ -42,14 +42,16 @@ inline rapidjson::Value doubleToJson( const double& val, rapidjson::Document& do // Helper function to convert doubles into json format recursively, by properly handlings NaNs, and Infs template<> -inline rapidjson::Value toJson( const double& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const double & val, const CspType & typ, rapidjson::Document & doc, + PyObject * callable ) { return doubleToJson( val, doc ); } // Helper function to convert Enums into json format recursively template<> -inline rapidjson::Value toJson( const CspEnum& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const CspEnum & val, const CspType & typ, rapidjson::Document & doc, + PyObject * callable ) { // NOTE: Assume that passed enum has greater lifetime than the json object return rapidjson::Value( rapidjson::StringRef( val.name() ) ); @@ -57,7 +59,8 @@ inline rapidjson::Value toJson( const CspEnum& val, const CspType& typ, rapidjso // Helper function to convert strings into json format recursively template<> -inline rapidjson::Value toJson( const std::string& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const std::string & val, const CspType & typ, rapidjson::Document & doc, + PyObject * callable ) { // NOTE: Assume that passed string has greater lifetime than the json object return rapidjson::Value( rapidjson::StringRef( val ) ); @@ -65,16 +68,18 @@ inline rapidjson::Value toJson( const std::string& val, const CspType& typ, rapi // Helper function to convert TimeDelta into json format recursively template<> -inline rapidjson::Value toJson( const TimeDelta& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const TimeDelta & val, const CspType & typ, rapidjson::Document & doc, + PyObject * callable ) { - if( val.isNone() ) return rapidjson::Value(); + if( val.isNone() ) + return rapidjson::Value(); // Convert TimeDelta to . // sign( 1 ) + seconds ( 18 ) + '.'( 1 ) + microseconds( 9 ) + '\0'( 1 ) - char buf[32] = {}; - long seconds = val.abs().asSeconds(); + char buf[32] = {}; + long seconds = val.abs().asSeconds(); auto microseconds = static_cast( val.abs().nanoseconds() / NANOS_PER_MICROSECOND ); - auto len = sprintf( buf, "%c%ld.%06u", ( val.sign() >= 0 ) ? '+' : '-', seconds, microseconds ); + auto len = sprintf( buf, "%c%ld.%06u", ( val.sign() >= 0 ) ? '+' : '-', seconds, microseconds ); rapidjson::Value res; res.SetString( buf, len, doc.GetAllocator() ); return res; @@ -82,14 +87,15 @@ inline rapidjson::Value toJson( const TimeDelta& val, const CspType& typ, rapidj // Helper function to convert Date into json format recursively template<> -inline rapidjson::Value toJson( const Date& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const Date & val, const CspType & typ, rapidjson::Document & doc, PyObject * callable ) { - if( val.isNone() ) return rapidjson::Value(); + if( val.isNone() ) + return rapidjson::Value(); // Convert Date to -- // year( 4 ) + '-'( 1 ) + month( 2 ) + '-'( 1 ) + day( 2 ) + '\0'( 1 ) char buf[32] = {}; - auto len = sprintf( buf, "%04u-%02u-%02u", val.year(), val.month(), val.day() ); + auto len = sprintf( buf, "%04u-%02u-%02u", val.year(), val.month(), val.day() ); rapidjson::Value res; res.SetString( buf, len, doc.GetAllocator() ); return res; @@ -97,15 +103,16 @@ inline rapidjson::Value toJson( const Date& val, const CspType& typ, rapidjson:: // Helper function to convert Time into json format recursively template<> -inline rapidjson::Value toJson( const Time& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const Time & val, const CspType & typ, rapidjson::Document & doc, PyObject * callable ) { - if( val.isNone() ) return rapidjson::Value(); + if( val.isNone() ) + return rapidjson::Value(); // Convert the Time to ::. // hours( 2 ) + ':'( 1 ) + minutes( 2 ) + ':'( 1 ) + seconds( 2 ) + '.'( 1 ) + micros( 6 ) + '\0'( 1 ) char buf[48] = {}; - auto micros = static_cast( val.nanosecond() / NANOS_PER_MICROSECOND ); - auto len = sprintf( buf, "%02u:%02u:%02u.%06u", val.hour(), val.minute(), val.second(), micros ); + auto micros = static_cast( val.nanosecond() / NANOS_PER_MICROSECOND ); + auto len = sprintf( buf, "%02u:%02u:%02u.%06u", val.hour(), val.minute(), val.second(), micros ); rapidjson::Value res; res.SetString( buf, len, doc.GetAllocator() ); return res; @@ -113,22 +120,23 @@ inline rapidjson::Value toJson( const Time& val, const CspType& typ, rapidjson:: // Helper function to convert DateTime into json format recursively template<> -inline rapidjson::Value toJson( const DateTime& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const DateTime & val, const CspType & typ, rapidjson::Document & doc, + PyObject * callable ) { - if( val.isNone() ) return rapidjson::Value(); + if( val.isNone() ) + return rapidjson::Value(); // Convert the datetime value into an ISO 8601 formatted string DateTimeEx dtx( val ); - char iso_str[80] = {}; + char iso_str[80] = {}; static const std::string utc_offset = "+00:00"; - auto micros = static_cast( dtx.nanoseconds() / NANOS_PER_MICROSECOND ); + auto micros = static_cast( dtx.nanoseconds() / NANOS_PER_MICROSECOND ); // Hardcode UTC since all times in csp are UTC // NOTE: Python's datetime.fromisoformat() API does not support nanoseconds in the string // Hence we truncate from nanos to micros to allow easy conversion to python datetime objects - auto len = sprintf( iso_str, "%04u-%02u-%02uT%02u:%02u:%02u.%06u%s", - dtx.year(), dtx.month(), dtx.day(), - dtx.hour(), dtx.minute(), dtx.second(), micros, utc_offset.c_str() ); + auto len = sprintf( iso_str, "%04u-%02u-%02uT%02u:%02u:%02u.%06u%s", dtx.year(), dtx.month(), dtx.day(), dtx.hour(), + dtx.minute(), dtx.second(), micros, utc_offset.c_str() ); rapidjson::Value res; res.SetString( iso_str, len, doc.GetAllocator() ); return res; @@ -136,14 +144,16 @@ inline rapidjson::Value toJson( const DateTime& val, const CspType& typ, rapidjs // Helper function to convert csp Structs into json format recursively template<> -inline rapidjson::Value toJson( const StructPtr& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const StructPtr & val, const CspType & typ, rapidjson::Document & doc, + PyObject * callable ) { return toJsonRecursive( val, doc, callable ); } // Helper function to convert python objects in csp Structs into json format recursively template<> -inline rapidjson::Value toJson( const DialectGenericType& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const DialectGenericType & val, const CspType & typ, rapidjson::Document & doc, + PyObject * callable ) { auto py_obj = PyObjectPtr::own( toPython( val ) ); return pyObjectToJson( py_obj.get(), doc, callable, false ); @@ -151,12 +161,13 @@ inline rapidjson::Value toJson( const DialectGenericType& val, const CspType& ty // Helper function to convert arrays in csp Structs into json lists recursively template -inline rapidjson::Value toJson( const std::vector& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +inline rapidjson::Value toJson( const std::vector & val, const CspType & typ, rapidjson::Document & doc, + PyObject * callable ) { using ElemT = typename CspType::Type::toCArrayElemType::type; - auto const * arrayType = static_cast( &typ ); - const CspType * elemType = arrayType -> elemType().get(); + auto const * arrayType = static_cast( &typ ); + const CspType * elemType = arrayType->elemType().get(); rapidjson::Value new_list; new_list.SetArray(); @@ -168,37 +179,40 @@ inline rapidjson::Value toJson( const std::vector& val, const CspType& return new_list; } -rapidjson::Value toJsonRecursive( const StructPtr& self, rapidjson::Document& doc, PyObject * callable ) +rapidjson::Value toJsonRecursive( const StructPtr & self, rapidjson::Document & doc, PyObject * callable ) { auto * struct_ptr = self.get(); if( struct_ptr == nullptr ) { CSP_THROW( ValueError, "Cannot call to_json on NULL struct object" ); } - auto * meta = static_cast( self -> meta() ); + auto * meta = static_cast( self->meta() ); rapidjson::Value new_dict; new_dict.SetObject(); - auto& fields = meta -> fields(); - for( const auto& field: fields ) + auto & fields = meta->fields(); + for( const auto & field : fields ) { - if( !field -> isSet( struct_ptr ) ) + if( !field->isSet( struct_ptr ) ) { continue; } - auto& key = field -> fieldname(); - auto sub_json = switchCspType( field -> type(), [field, struct_ptr, callable, &doc]( auto tag ) - { - using CType = typename decltype( tag )::type; - auto *typedField = static_cast::type *>( field.get() ); - return toJson( typedField -> value( struct_ptr ), *field -> type(), doc, callable ); - } ); + auto & key = field->fieldname(); + auto sub_json + = switchCspType( field->type(), + [field, struct_ptr, callable, &doc]( auto tag ) + { + using CType = typename decltype( tag )::type; + auto * typedField + = static_cast::type *>( field.get() ); + return toJson( typedField->value( struct_ptr ), *field->type(), doc, callable ); + } ); new_dict.AddMember( rapidjson::StringRef( key ), sub_json, doc.GetAllocator() ); } return new_dict; } -rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc ) +rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document & doc ) { // Only support strings, ints, and floats as keys // JSON encoding requires all names to be strings so convert them to strings @@ -216,12 +230,12 @@ rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc ) } else if( PyFloat_Check( py_key ) ) { - auto key = PyFloat_AsDouble( py_key ); + auto key = PyFloat_AsDouble( py_key ); auto json_obj = doubleToJson( key, doc ); - if ( json_obj.IsNull() ) + if( json_obj.IsNull() ) { - auto * str_obj = PyObject_Str( py_key ); - Py_ssize_t len = 0; + auto * str_obj = PyObject_Str( py_key ); + Py_ssize_t len = 0; const char * str = PyUnicode_AsUTF8AndSize( str_obj, &len ); CSP_THROW( ValueError, "Cannot serialize " + std::string( str ) + " to key in JSON" ); } @@ -235,14 +249,14 @@ rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc ) } else { - CSP_THROW( ValueError, "Cannot serialize key of type: " + std::string( Py_TYPE( py_key ) -> tp_name ) ); + CSP_THROW( ValueError, "Cannot serialize key of type: " + std::string( Py_TYPE( py_key )->tp_name ) ); // Never reaches here } return val; } // Helper function to parse python lists into json arrays recursively -rapidjson::Value pyListToJson( PyObject * py_list, rapidjson::Document& doc, PyObject * callable ) +rapidjson::Value pyListToJson( PyObject * py_list, rapidjson::Document & doc, PyObject * callable ) { size_t size = PyList_GET_SIZE( py_list ); rapidjson::Value new_list; @@ -251,14 +265,14 @@ rapidjson::Value pyListToJson( PyObject * py_list, rapidjson::Document& doc, PyO for( size_t idx = 0; idx < size; ++idx ) { auto * item = PyList_GET_ITEM( py_list, idx ); - auto res = pyObjectToJson( item, doc, callable, false ); + auto res = pyObjectToJson( item, doc, callable, false ); new_list.PushBack( res, doc.GetAllocator() ); } return new_list; } // Helper function to parse python tuples into json arrays recursively -rapidjson::Value pyTupleToJson( PyObject * py_tuple, rapidjson::Document& doc, PyObject * callable ) +rapidjson::Value pyTupleToJson( PyObject * py_tuple, rapidjson::Document & doc, PyObject * callable ) { size_t size = PyTuple_GET_SIZE( py_tuple ); rapidjson::Value new_list; @@ -267,18 +281,18 @@ rapidjson::Value pyTupleToJson( PyObject * py_tuple, rapidjson::Document& doc, P for( size_t idx = 0; idx < size; ++idx ) { auto * item = PyTuple_GetItem( py_tuple, idx ); - auto res = pyObjectToJson( item, doc, callable, false ); + auto res = pyObjectToJson( item, doc, callable, false ); new_list.PushBack( res, doc.GetAllocator() ); } return new_list; } // Helper function to parse python dicts into json objects recursively -rapidjson::Value pyDictToJson( PyObject * py_dict, rapidjson::Document& doc, PyObject * callable ) +rapidjson::Value pyDictToJson( PyObject * py_dict, rapidjson::Document & doc, PyObject * callable ) { - PyObject * py_key = NULL; + PyObject * py_key = NULL; PyObject * py_value = NULL; - Py_ssize_t pos = 0; + Py_ssize_t pos = 0; rapidjson::Value new_dict; new_dict.SetObject(); @@ -291,7 +305,7 @@ rapidjson::Value pyDictToJson( PyObject * py_dict, rapidjson::Document& doc, PyO return new_dict; } -rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyObject * callable, bool is_recursing ) +rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document & doc, PyObject * callable, bool is_recursing ) { INIT_PYDATETIME; if( value == Py_None ) @@ -321,7 +335,7 @@ rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyO else if( PyBytes_Check( value ) ) { Py_ssize_t len = PyBytes_Size( value ); - auto str = PyBytes_AsString( value ); + auto str = PyBytes_AsString( value ); rapidjson::Value str_val; str_val.SetString( str, len, doc.GetAllocator() ); return str_val; @@ -363,12 +377,12 @@ rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyO } else if( PyType_IsSubtype( Py_TYPE( value ), &PyStruct::PyType ) ) { - auto struct_ptr = static_cast( value ) -> struct_; + auto struct_ptr = static_cast( value )->struct_; return toJsonRecursive( struct_ptr, doc, callable ); } else if( PyType_IsSubtype( Py_TYPE( value ), &PyCspEnum::PyType ) ) { - auto enum_ptr = static_cast( value ) -> enum_; + auto enum_ptr = static_cast( value )->enum_; return toJson( enum_ptr, CspType( CspType::Type::ENUM ), doc, callable ); } else @@ -377,7 +391,7 @@ rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyO { // Looks like we are recursively calling pyObjectToJson for a generic unsupported type // just return the object since we don't know how to jsonify it - CSP_THROW( ValueError, "Cannot serialize value of type: " + std::string( Py_TYPE( value ) -> tp_name ) ); + CSP_THROW( ValueError, "Cannot serialize value of type: " + std::string( Py_TYPE( value )->tp_name ) ); // Never reaches here } else @@ -407,32 +421,22 @@ class StringHolder { public: typedef char Ch; - StringHolder( std::string& s ) : s_( s ) + StringHolder( std::string & s ) + : s_( s ) { s_.reserve( DEFAULT_BUFFER_SIZE ); } - void Put( char c ) - { - s_.push_back( c ); - } - void Clear() - { - s_.clear(); - } - void Flush() - { - return; - } - size_t Size() const - { - return s_.length(); - } + void Put( char c ) { s_.push_back( c ); } + void Clear() { s_.clear(); } + void Flush() { return; } + size_t Size() const { return s_.length(); } const size_t DEFAULT_BUFFER_SIZE = 4096; + private: - std::string& s_; + std::string & s_; }; -std::string structToJson( const StructPtr& struct_ptr, PyObject * callable ) +std::string structToJson( const StructPtr & struct_ptr, PyObject * callable ) { // Need this just for the Allocator rapidjson::Document placeholder_doc; @@ -445,4 +449,4 @@ std::string structToJson( const StructPtr& struct_ptr, PyObject * callable ) return output; } -} +} // namespace csp::python diff --git a/cpp/csp/python/PyTimerAdapter.cpp b/cpp/csp/python/PyTimerAdapter.cpp index e8d8e007..8a42639b 100644 --- a/cpp/csp/python/PyTimerAdapter.cpp +++ b/cpp/csp/python/PyTimerAdapter.cpp @@ -7,11 +7,12 @@ namespace csp::python { -static InputAdapter * timer_creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, PushMode pushMode, PyObject * args ) +static InputAdapter * timer_creator( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, + PushMode pushMode, PyObject * args ) { PyObject * pyInterval = nullptr; PyObject * pyValue = nullptr; - int allowDeviation; + int allowDeviation; if( !PyArg_ParseTuple( args, "OOp", &pyInterval, &pyValue, &allowDeviation ) ) CSP_THROW( PythonPassthrough, "" ); @@ -20,16 +21,16 @@ static InputAdapter * timer_creator( csp::AdapterManager * manager, PyEngine * p auto cspType = pyTypeAsCspType( pyType ); - return switchCspType( cspType, - [ engine = pyengine -> engine(), &cspType, interval, pyValue, allowDeviation ]( - auto tag ) -> InputAdapter * - { - using T = typename decltype(tag)::type; - return engine -> createOwnedObject>( - cspType, interval, fromPython( pyValue, *cspType ), bool( allowDeviation ) ); - } ); + return switchCspType( + cspType, + [engine = pyengine->engine(), &cspType, interval, pyValue, allowDeviation]( auto tag ) -> InputAdapter * + { + using T = typename decltype( tag )::type; + return engine->createOwnedObject>( + cspType, interval, fromPython( pyValue, *cspType ), bool( allowDeviation ) ); + } ); } REGISTER_INPUT_ADAPTER( _timer, timer_creator ); -} +} // namespace csp::python diff --git a/cpp/csp/python/adapters/kafkaadapterimpl.cpp b/cpp/csp/python/adapters/kafkaadapterimpl.cpp index 3c88382e..950d8698 100644 --- a/cpp/csp/python/adapters/kafkaadapterimpl.cpp +++ b/cpp/csp/python/adapters/kafkaadapterimpl.cpp @@ -13,67 +13,60 @@ using namespace csp::adapters::kafka; namespace csp::python { -//AdapterManager +// AdapterManager csp::AdapterManager * create_kafka_adapter_manager( PyEngine * engine, const Dictionary & properties ) { - return engine -> engine() -> createOwnedObject( properties ); + return engine->engine()->createOwnedObject( properties ); } -static InputAdapter * create_kafka_input_adapter( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, PushMode pushMode, PyObject * args ) +static InputAdapter * create_kafka_input_adapter( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, + PushMode pushMode, PyObject * args ) { auto & cspType = pyTypeAsCspType( pyType ); PyObject * pyProperties; PyObject * type; - auto * kafkaManager = dynamic_cast( manager ); + auto * kafkaManager = dynamic_cast( manager ); if( !kafkaManager ) CSP_THROW( TypeError, "Expected KafkaAdapterManager" ); - if( !PyArg_ParseTuple( args, "O!O!", - &PyType_Type, &type, - &PyDict_Type, &pyProperties ) ) + if( !PyArg_ParseTuple( args, "O!O!", &PyType_Type, &type, &PyDict_Type, &pyProperties ) ) CSP_THROW( PythonPassthrough, "" ); - return kafkaManager -> getInputAdapter( cspType, pushMode, fromPython( pyProperties ) ); + return kafkaManager->getInputAdapter( cspType, pushMode, fromPython( pyProperties ) ); } -static OutputAdapter * create_kafka_output_adapter( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * args ) +static OutputAdapter * create_kafka_output_adapter( csp::AdapterManager * manager, PyEngine * pyengine, + PyObject * args ) { PyObject * pyProperties; PyObject * pyType; - auto * kafkaManager = dynamic_cast( manager ); + auto * kafkaManager = dynamic_cast( manager ); if( !kafkaManager ) CSP_THROW( TypeError, "Expected KafkaAdapterManager" ); - if( !PyArg_ParseTuple( args, "OO!", - &pyType, - &PyDict_Type, &pyProperties ) ) + if( !PyArg_ParseTuple( args, "OO!", &pyType, &PyDict_Type, &pyProperties ) ) CSP_THROW( PythonPassthrough, "" ); auto & cspType = pyTypeAsCspType( pyType ); - return kafkaManager -> getOutputAdapter( cspType, fromPython( pyProperties ) ); + return kafkaManager->getOutputAdapter( cspType, fromPython( pyProperties ) ); } REGISTER_ADAPTER_MANAGER( _kafka_adapter_manager, create_kafka_adapter_manager ); -REGISTER_INPUT_ADAPTER( _kafka_input_adapter, create_kafka_input_adapter ); -REGISTER_OUTPUT_ADAPTER( _kafka_output_adapter, create_kafka_output_adapter ); - -static PyModuleDef _kafkaadapterimpl_module = { - PyModuleDef_HEAD_INIT, - "_kafkaadapterimpl", - "_kafkaadapterimpl c++ module", - -1, - NULL, NULL, NULL, NULL, NULL -}; - -PyMODINIT_FUNC PyInit__kafkaadapterimpl(void) +REGISTER_INPUT_ADAPTER( _kafka_input_adapter, create_kafka_input_adapter ); +REGISTER_OUTPUT_ADAPTER( _kafka_output_adapter, create_kafka_output_adapter ); + +static PyModuleDef _kafkaadapterimpl_module + = { PyModuleDef_HEAD_INIT, "_kafkaadapterimpl", "_kafkaadapterimpl c++ module", -1, NULL, NULL, NULL, NULL, NULL }; + +PyMODINIT_FUNC PyInit__kafkaadapterimpl( void ) { - PyObject* m; + PyObject * m; - m = PyModule_Create( &_kafkaadapterimpl_module); + m = PyModule_Create( &_kafkaadapterimpl_module ); if( m == NULL ) return NULL; @@ -83,4 +76,4 @@ PyMODINIT_FUNC PyInit__kafkaadapterimpl(void) return m; } -} +} // namespace csp::python diff --git a/cpp/csp/python/adapters/parquetadapterimpl.cpp b/cpp/csp/python/adapters/parquetadapterimpl.cpp index a7f41aae..77bf81d6 100644 --- a/cpp/csp/python/adapters/parquetadapterimpl.cpp +++ b/cpp/csp/python/adapters/parquetadapterimpl.cpp @@ -1,31 +1,31 @@ -#include +#include +#include +#include +#include +#include #include #include -#include #include #include +#include +#include #include #include #include #include +#include #include +#include #include #include -#include -#include -#include -#include #include -#include -#include -#include -#include +#include #include -#include +#include using namespace csp::adapters::parquet; -//using namespace csp::cppnodes; -//namespace csp::adapters::parquet +// using namespace csp::cppnodes; +// namespace csp::adapters::parquet namespace csp::cppnodes { DECLARE_CPPNODE( parquet_dict_basket_writer ) @@ -35,42 +35,42 @@ DECLARE_CPPNODE( parquet_dict_basket_writer ) TS_DICTBASKET_INPUT( Generic, input ); TS_INPUT( std::string, filename_provider ); - - STATE_VAR( csp::adapters::parquet::ParquetDictBasketOutputWriter*, s_outputWriter ); + STATE_VAR( csp::adapters::parquet::ParquetDictBasketOutputWriter *, s_outputWriter ); INIT_CPPNODE( parquet_dict_basket_writer ) { - const csp::python::PyObjectPtr *writerObjectPtr = reinterpret_cast(&writer); + const csp::python::PyObjectPtr * writerObjectPtr + = reinterpret_cast( &writer ); auto managerObjectPtr = csp::python::PyObjectPtr::incref( - PyObject_CallMethod( writerObjectPtr -> get(), "_get_output_adapter_manager", "" ) ); + PyObject_CallMethod( writerObjectPtr->get(), "_get_output_adapter_manager", "" ) ); - auto *outputAdapterManager = dynamic_cast( csp::python::PyAdapterManagerWrapper::extractAdapterManager( - managerObjectPtr.get() )); + auto * outputAdapterManager = dynamic_cast( + csp::python::PyAdapterManagerWrapper::extractAdapterManager( managerObjectPtr.get() ) ); - s_outputWriter = outputAdapterManager -> createDictOutputBasketWriter( column_name.value().c_str(), input.type() ); + s_outputWriter + = outputAdapterManager->createDictOutputBasketWriter( column_name.value().c_str(), input.type() ); } INVOKE() { if( unlikely( filename_provider.ticked() ) ) { - s_outputWriter -> onFileNameChange( filename_provider.lastValue() ); + s_outputWriter->onFileNameChange( filename_provider.lastValue() ); } - if( s_outputWriter -> isFileOpen() ) + if( s_outputWriter->isFileOpen() ) { - const auto &shape = input.shape(); - for( auto &&tickedInputIt = input.tickedinputs(); tickedInputIt; ++tickedInputIt ) + const auto & shape = input.shape(); + for( auto && tickedInputIt = input.tickedinputs(); tickedInputIt; ++tickedInputIt ) { - s_outputWriter -> writeValue( shape[ tickedInputIt.elemId() ], tickedInputIt.get() ); + s_outputWriter->writeValue( shape[tickedInputIt.elemId()], tickedInputIt.get() ); } } } }; EXPORT_CPPNODE( parquet_dict_basket_writer ); -} - +} // namespace csp::cppnodes REGISTER_CPPNODE( csp::cppnodes, parquet_dict_basket_writer ); @@ -95,15 +95,14 @@ struct PyArrowInitializer class FileNameGenerator : public csp::Generator { public: - - FileNameGenerator( PyObject *wrappedGenerator ) - : m_wrappedGenerator( csp::python::PyObjectPtr::incref( wrappedGenerator ) ) + FileNameGenerator( PyObject * wrappedGenerator ) + : m_wrappedGenerator( csp::python::PyObjectPtr::incref( wrappedGenerator ) ) { } void init( csp::DateTime start, csp::DateTime end ) override { - PyObject *tp = PyTuple_New( 2 ); + PyObject * tp = PyTuple_New( 2 ); if( !tp ) CSP_THROW( csp::python::PythonPassthrough, "" ); @@ -114,7 +113,7 @@ class FileNameGenerator : public csp::Generator, csp::DateTime, csp::DateTime> { public: - ArrowTableGenerator( PyObject *wrappedGenerator ) - : m_wrappedGenerator( csp::python::PyObjectPtr::incref( wrappedGenerator ) ) + ArrowTableGenerator( PyObject * wrappedGenerator ) + : m_wrappedGenerator( csp::python::PyObjectPtr::incref( wrappedGenerator ) ) { PyArrowInitializer::ensureInitialized(); } void init( csp::DateTime start, csp::DateTime end ) override { - PyObject *tp = PyTuple_New( 2 ); + PyObject * tp = PyTuple_New( 2 ); if( !tp ) CSP_THROW( csp::python::PythonPassthrough, "" ); @@ -161,7 +159,7 @@ class ArrowTableGenerator : public csp::Generator, "Parquet file generator expected to return iterator" ); } - virtual bool next( std::shared_ptr &value ) override + virtual bool next( std::shared_ptr & value ) override { if( m_iter.ptr() == nullptr ) { @@ -178,24 +176,27 @@ class ArrowTableGenerator : public csp::Generator, return false; } - if(!PyBytes_Check( nextVal )) + if( !PyBytes_Check( nextVal ) ) { - CSP_THROW( csp::TypeError, "Invalid arrow buffer type, expected bytes got " << Py_TYPE( nextVal ) -> tp_name ); + CSP_THROW( csp::TypeError, + "Invalid arrow buffer type, expected bytes got " << Py_TYPE( nextVal )->tp_name ); } const char * data = PyBytes_AsString( nextVal ); if( !data ) CSP_THROW( csp::python::PythonPassthrough, "" ); - auto size = PyBytes_Size(nextVal); - m_data = csp::python::PyObjectPtr::incref(nextVal); - std::shared_ptr bufferReader = std::make_shared( - reinterpret_cast(data), size ); - std::shared_ptr reader = arrow::ipc::RecordBatchStreamReader::Open(bufferReader.get()).ValueOrDie(); + auto size = PyBytes_Size( nextVal ); + m_data = csp::python::PyObjectPtr::incref( nextVal ); + std::shared_ptr bufferReader + = std::make_shared( reinterpret_cast( data ), size ); + std::shared_ptr reader + = arrow::ipc::RecordBatchStreamReader::Open( bufferReader.get() ).ValueOrDie(); auto result = reader->ToTable(); - if (!(result.ok())) - CSP_THROW(csp::RuntimeException, "Failed read arrow table from buffer"); - value = std::move(result.ValueUnsafe()); + if( !( result.ok() ) ) + CSP_THROW( csp::RuntimeException, "Failed read arrow table from buffer" ); + value = std::move( result.ValueUnsafe() ); return true; } + private: csp::python::PyObjectPtr m_wrappedGenerator; csp::python::PyObjectPtr m_iter; @@ -203,237 +204,259 @@ class ArrowTableGenerator : public csp::Generator, csp::python::PyObjectPtr m_data; }; -template< typename CspCType> +template class NumpyArrayWriterImpl : public TypedDialectGenericListWriterInterface { public: - NumpyArrayWriterImpl( PyArray_Descr *expectedArrayDesc ) - : m_expectedArrayDesc( expectedArrayDesc ) + NumpyArrayWriterImpl( PyArray_Descr * expectedArrayDesc ) + : m_expectedArrayDesc( expectedArrayDesc ) { } - void writeItems( const csp::DialectGenericType &listObject ) override + void writeItems( const csp::DialectGenericType & listObject ) override { - PyObject *object = csp::python::toPythonBorrowed( listObject ); + PyObject * object = csp::python::toPythonBorrowed( listObject ); if( !PyArray_Check( object ) ) { - CSP_THROW( csp::TypeError, "While writing to parquet expected numpy array type, got " << Py_TYPE( object ) -> tp_name ); + CSP_THROW( csp::TypeError, + "While writing to parquet expected numpy array type, got " << Py_TYPE( object )->tp_name ); } - PyArrayObject *arrayObject = ( PyArrayObject * ) ( object ); - char npy_type = PyArray_DESCR( arrayObject ) -> type; - if( PyArray_DESCR( arrayObject ) -> kind != m_expectedArrayDesc -> kind ) + PyArrayObject * arrayObject = (PyArrayObject *)( object ); + char npy_type = PyArray_DESCR( arrayObject )->type; + if( PyArray_DESCR( arrayObject )->kind != m_expectedArrayDesc->kind ) { - CSP_THROW( csp::TypeError, - "Expected array of type " << csp::python::PyObjectPtr::own( PyObject_Repr( ( PyObject * ) m_expectedArrayDesc ) ) - << " got " - << csp::python::PyObjectPtr::own( PyObject_Repr( ( PyObject * ) PyArray_DESCR( arrayObject ) ) ) ); + CSP_THROW( + csp::TypeError, + "Expected array of type " + << csp::python::PyObjectPtr::own( PyObject_Repr( (PyObject *)m_expectedArrayDesc ) ) << " got " + << csp::python::PyObjectPtr::own( PyObject_Repr( (PyObject *)PyArray_DESCR( arrayObject ) ) ) ); } auto ndim = PyArray_NDIM( arrayObject ); - CSP_TRUE_OR_THROW_RUNTIME( ndim == 1, "While writing to parquet expected numpy array with 1 dimension" << " got " << ndim ); + CSP_TRUE_OR_THROW_RUNTIME( + ndim == 1, "While writing to parquet expected numpy array with 1 dimension" << " got " << ndim ); switch( npy_type ) { - case NPY_BYTELTR: writeValues( arrayObject ); break; - case NPY_UBYTELTR: writeValues( arrayObject ); break; - case NPY_SHORTLTR: writeValues( arrayObject ); break; - case NPY_USHORTLTR: writeValues( arrayObject ); break; - case NPY_INTLTR: writeValues( arrayObject ); break; - case NPY_UINTLTR: writeValues( arrayObject ); break; - case NPY_LONGLTR: writeValues( arrayObject ); break; - case NPY_ULONGLTR: writeValues( arrayObject ); break; - case NPY_LONGLONGLTR: writeValues( arrayObject ); break; - case NPY_ULONGLONGLTR: writeValues( arrayObject ); break; - - case NPY_FLOATLTR: writeValues( arrayObject ); break; - case NPY_DOUBLELTR: writeValues( arrayObject ); break; + case NPY_BYTELTR: + writeValues( arrayObject ); + break; + case NPY_UBYTELTR: + writeValues( arrayObject ); + break; + case NPY_SHORTLTR: + writeValues( arrayObject ); + break; + case NPY_USHORTLTR: + writeValues( arrayObject ); + break; + case NPY_INTLTR: + writeValues( arrayObject ); + break; + case NPY_UINTLTR: + writeValues( arrayObject ); + break; + case NPY_LONGLTR: + writeValues( arrayObject ); + break; + case NPY_ULONGLTR: + writeValues( arrayObject ); + break; + case NPY_LONGLONGLTR: + writeValues( arrayObject ); + break; + case NPY_ULONGLONGLTR: + writeValues( arrayObject ); + break; + + case NPY_FLOATLTR: + writeValues( arrayObject ); + break; + case NPY_DOUBLELTR: + writeValues( arrayObject ); + break; default: writeValues( arrayObject ); } } + private: template void writeValues( PyArrayObject * arrayObject ) { - auto arraySize = PyArray_Size( ( PyObject * ) arrayObject ); - if( PyArray_ISCARRAY_RO(arrayObject) ) + auto arraySize = PyArray_Size( (PyObject *)arrayObject ); + if( PyArray_ISCARRAY_RO( arrayObject ) ) { - NumpyCType* data = reinterpret_cast( PyArray_DATA( arrayObject ) ); - for (decltype(arraySize) i = 0; i < arraySize; ++i) + NumpyCType * data = reinterpret_cast( PyArray_DATA( arrayObject ) ); + for( decltype( arraySize ) i = 0; i < arraySize; ++i ) { - this->writeValue(static_cast(data[i])); + this->writeValue( static_cast( data[i] ) ); } } else { - for (decltype(arraySize) i = 0; i < arraySize; ++i) + for( decltype( arraySize ) i = 0; i < arraySize; ++i ) { - this->writeValue(static_cast(*reinterpret_cast(PyArray_GETPTR1(arrayObject, i)))); + this->writeValue( + static_cast( *reinterpret_cast( PyArray_GETPTR1( arrayObject, i ) ) ) ); } } } - PyArray_Descr *m_expectedArrayDesc; + PyArray_Descr * m_expectedArrayDesc; }; class NumpyUnicodeArrayWriter : public TypedDialectGenericListWriterInterface { public: - NumpyUnicodeArrayWriter() - { - } + NumpyUnicodeArrayWriter() {} - void writeItems( const csp::DialectGenericType &listObject ) override + void writeItems( const csp::DialectGenericType & listObject ) override { - PyObject *object = csp::python::toPythonBorrowed( listObject ); + PyObject * object = csp::python::toPythonBorrowed( listObject ); if( !PyArray_Check( object ) ) { - CSP_THROW( csp::TypeError, "While writing to parquet expected numpy array type, got " << Py_TYPE( object ) -> tp_name ); + CSP_THROW( csp::TypeError, + "While writing to parquet expected numpy array type, got " << Py_TYPE( object )->tp_name ); } - PyArrayObject *arrayObject = ( PyArrayObject * ) ( object ); + PyArrayObject * arrayObject = (PyArrayObject *)( object ); - if( PyArray_DESCR( arrayObject ) -> type_num != NPY_UNICODE ) + if( PyArray_DESCR( arrayObject )->type_num != NPY_UNICODE ) { - CSP_THROW( csp::TypeError, - "Expected array of type " << csp::python::PyObjectPtr::own( PyObject_Repr( ( PyObject * ) m_expectedArrayDesc ) ) - << " got " - << csp::python::PyObjectPtr::own( - PyObject_Repr( ( PyObject * ) PyArray_DESCR( arrayObject ) ) ) ); + CSP_THROW( + csp::TypeError, + "Expected array of type " + << csp::python::PyObjectPtr::own( PyObject_Repr( (PyObject *)m_expectedArrayDesc ) ) << " got " + << csp::python::PyObjectPtr::own( PyObject_Repr( (PyObject *)PyArray_DESCR( arrayObject ) ) ) ); } - auto elementSize = PyArray_DESCR( arrayObject ) -> elsize; + auto elementSize = PyArray_DESCR( arrayObject )->elsize; auto ndim = PyArray_NDIM( arrayObject ); - CSP_TRUE_OR_THROW_RUNTIME( ndim == 1, "While writing to parquet expected numpy array with 1 dimension" << " got " << ndim ); - std::wstring_convert,char32_t> converter; + CSP_TRUE_OR_THROW_RUNTIME( + ndim == 1, "While writing to parquet expected numpy array with 1 dimension" << " got " << ndim ); + std::wstring_convert, char32_t> converter; auto arraySize = PyArray_Size( object ); if( PyArray_ISCARRAY_RO( arrayObject ) ) { - auto data = reinterpret_cast(PyArray_DATA( arrayObject )); + auto data = reinterpret_cast( PyArray_DATA( arrayObject ) ); for( decltype( arraySize ) i = 0; i < arraySize; ++i ) { - std::string value = converter.to_bytes( reinterpret_cast(data + elementSize * i), - reinterpret_cast(data + elementSize * ( i + 1 )) ); - this -> writeValue( value ); + std::string value + = converter.to_bytes( reinterpret_cast( data + elementSize * i ), + reinterpret_cast( data + elementSize * ( i + 1 ) ) ); + this->writeValue( value ); } } else { for( decltype( arraySize ) i = 0; i < arraySize; ++i ) { - char *elementPtr = reinterpret_cast(PyArray_GETPTR1( arrayObject, i )); - std::string value = converter.to_bytes( reinterpret_cast(elementPtr), - reinterpret_cast(elementPtr + elementSize ) ); - this -> writeValue( value ); + char * elementPtr = reinterpret_cast( PyArray_GETPTR1( arrayObject, i ) ); + std::string value = converter.to_bytes( reinterpret_cast( elementPtr ), + reinterpret_cast( elementPtr + elementSize ) ); + this->writeValue( value ); } } } private: - PyArray_Descr *m_expectedArrayDesc; + PyArray_Descr * m_expectedArrayDesc; }; -static inline DialectGenericListWriterInterface::Ptr create_numpy_array_writer_impl( const csp::CspTypePtr &type ) +static inline DialectGenericListWriterInterface::Ptr create_numpy_array_writer_impl( const csp::CspTypePtr & type ) { try { - return csp::PartialSwitchCspType::invoke( - type.get(), - []( auto tag ) -> DialectGenericListWriterInterface::Ptr - { - using CValueType = typename decltype( tag )::type; - auto numpy_dtype = PyArray_DescrFromType( csp::python::NPY_TYPE::value ); - - if constexpr (std::is_same_v) - { - return std::make_shared(); - } - else - { - return std::make_shared>(numpy_dtype); - } - } - ); - } - catch( csp::TypeError &e ) - { - CSP_THROW( csp::TypeError, "Unsupported array value type when writing to parquet:" << type -> type().asString() ); + return csp::PartialSwitchCspType< + csp::CspType::Type::DOUBLE, csp::CspType::Type::INT64, csp::CspType::Type::BOOL, + csp::CspType::Type::STRING>::invoke( type.get(), + []( auto tag ) -> DialectGenericListWriterInterface::Ptr + { + using CValueType = typename decltype( tag )::type; + auto numpy_dtype = PyArray_DescrFromType( + csp::python::NPY_TYPE::value ); + + if constexpr( std::is_same_v ) + { + return std::make_shared(); + } + else + { + return std::make_shared>( + numpy_dtype ); + } + } ); + } + catch( csp::TypeError & e ) + { + CSP_THROW( csp::TypeError, "Unsupported array value type when writing to parquet:" << type->type().asString() ); } } - -template< typename V > +template class NumpyArrayReaderImpl final : public TypedDialectGenericListReaderInterface { public: - NumpyArrayReaderImpl( PyArray_Descr *expectedArrayDesc ) - : m_expectedArrayDesc( expectedArrayDesc ) + NumpyArrayReaderImpl( PyArray_Descr * expectedArrayDesc ) + : m_expectedArrayDesc( expectedArrayDesc ) { } - virtual csp::DialectGenericType create(uint32_t size) override + virtual csp::DialectGenericType create( uint32_t size ) override { npy_intp iSize = size; - Py_INCREF(m_expectedArrayDesc); - PyObject* arr = PyArray_SimpleNewFromDescr( 1, &iSize, m_expectedArrayDesc ); + Py_INCREF( m_expectedArrayDesc ); + PyObject * arr = PyArray_SimpleNewFromDescr( 1, &iSize, m_expectedArrayDesc ); // Since arr already has reference count - csp::python::PyObjectPtr objectPtr{csp::python::PyObjectPtr::own(arr)}; + csp::python::PyObjectPtr objectPtr{ csp::python::PyObjectPtr::own( arr ) }; // We need to make sure that's the case, since we are going to return pointer to raw buffer - CSP_ASSERT(PyArray_ISCARRAY( reinterpret_cast(arr))); + CSP_ASSERT( PyArray_ISCARRAY( reinterpret_cast( arr ) ) ); - csp::DialectGenericType res{csp::python::fromPython(arr)}; + csp::DialectGenericType res{ csp::python::fromPython( arr ) }; return res; } - csp::DialectGenericType create( uint32_t size, uint32_t maxElementSize ) override - { - CSP_NOT_IMPLEMENTED; - } + csp::DialectGenericType create( uint32_t size, uint32_t maxElementSize ) override { CSP_NOT_IMPLEMENTED; } - virtual V *getRawDataBuffer( const csp::DialectGenericType &list ) const override + virtual V * getRawDataBuffer( const csp::DialectGenericType & list ) const override { - auto arrayObject = reinterpret_cast(csp::python::toPythonBorrowed(list)); - return reinterpret_cast(PyArray_DATA( arrayObject )); + auto arrayObject = reinterpret_cast( csp::python::toPythonBorrowed( list ) ); + return reinterpret_cast( PyArray_DATA( arrayObject ) ); } - virtual void setValue(const csp::DialectGenericType& list, int index, const V& value) override + virtual void setValue( const csp::DialectGenericType & list, int index, const V & value ) override { - getRawDataBuffer(list)[index] = value; + getRawDataBuffer( list )[index] = value; } private: - PyArray_Descr *m_expectedArrayDesc; + PyArray_Descr * m_expectedArrayDesc; }; class NumpyUnicodeReaderImpl final : public TypedDialectGenericListReaderInterface { public: - NumpyUnicodeReaderImpl() - { - } + NumpyUnicodeReaderImpl() {} - virtual csp::DialectGenericType create( uint32_t size ) override - { - CSP_NOT_IMPLEMENTED; - } + virtual csp::DialectGenericType create( uint32_t size ) override { CSP_NOT_IMPLEMENTED; } csp::DialectGenericType create( uint32_t size, uint32_t maxElementSize ) override { npy_intp iSize = size; - PyArray_Descr *typ; - PyObject *type_string_descr = csp::python::toPython( std::string( "U" ) + std::to_string( maxElementSize ) ); + PyArray_Descr * typ; + PyObject * type_string_descr = csp::python::toPython( std::string( "U" ) + std::to_string( maxElementSize ) ); PyArray_DescrConverter( type_string_descr, &typ ); Py_DECREF( type_string_descr ); - PyObject *arr = PyArray_SimpleNewFromDescr( 1, &iSize, typ ); + PyObject * arr = PyArray_SimpleNewFromDescr( 1, &iSize, typ ); // Since arr already has reference count csp::python::PyObjectPtr objectPtr{ csp::python::PyObjectPtr::own( arr ) }; @@ -442,227 +465,221 @@ class NumpyUnicodeReaderImpl final : public TypedDialectGenericListReaderInterfa return res; } - std::string *getRawDataBuffer( const csp::DialectGenericType &list ) const override - { - return nullptr; - } + std::string * getRawDataBuffer( const csp::DialectGenericType & list ) const override { return nullptr; } - void setValue( const csp::DialectGenericType &list, int index, const std::string &value ) override + void setValue( const csp::DialectGenericType & list, int index, const std::string & value ) override { - auto arrayObject = reinterpret_cast(csp::python::toPythonBorrowed( list )); - std::wstring_convert,char32_t> converter; - auto elementSize = PyArray_DESCR( arrayObject ) -> elsize; - auto wideValue = converter.from_bytes( value ); - auto nElementsToCopy = std::min( int(elementSize / sizeof(char32_t)), int( wideValue.size() + 1 ) ); - std::copy_n( wideValue.c_str(), nElementsToCopy, reinterpret_cast(PyArray_GETPTR1( arrayObject, index )) ); + auto arrayObject = reinterpret_cast( csp::python::toPythonBorrowed( list ) ); + std::wstring_convert, char32_t> converter; + auto elementSize = PyArray_DESCR( arrayObject )->elsize; + auto wideValue = converter.from_bytes( value ); + auto nElementsToCopy = std::min( int( elementSize / sizeof( char32_t ) ), int( wideValue.size() + 1 ) ); + std::copy_n( wideValue.c_str(), nElementsToCopy, + reinterpret_cast( PyArray_GETPTR1( arrayObject, index ) ) ); } }; - -inline DialectGenericListReaderInterface::Ptr create_numpy_array_reader_impl( const csp::CspTypePtr &type ) +inline DialectGenericListReaderInterface::Ptr create_numpy_array_reader_impl( const csp::CspTypePtr & type ) { try { - return csp::PartialSwitchCspType::invoke( type.get(), - []( auto tag ) -> DialectGenericListReaderInterface::Ptr - { - using TagType = decltype(tag); - using CValueType = typename TagType::type; - auto numpy_dtype = PyArray_DescrFromType( - csp::python::NPY_TYPE::value ); - - if( numpy_dtype -> type_num == NPY_UNICODE ) - { - return std::make_shared(); - } - else - { - return std::make_shared>( - numpy_dtype ); - } - } - ); - } - catch( csp::TypeError &e ) - { - CSP_THROW( csp::TypeError, "Unsupported array value type when reading from parquet:" << type -> type().asString() ); + return csp::PartialSwitchCspType< + csp::CspType::Type::DOUBLE, csp::CspType::Type::INT64, csp::CspType::Type::BOOL, + csp::CspType::Type::STRING>::invoke( type.get(), + []( auto tag ) -> DialectGenericListReaderInterface::Ptr + { + using TagType = decltype( tag ); + using CValueType = typename TagType::type; + auto numpy_dtype = PyArray_DescrFromType( + csp::python::NPY_TYPE::value ); + + if( numpy_dtype->type_num == NPY_UNICODE ) + { + return std::make_shared(); + } + else + { + return std::make_shared>( + numpy_dtype ); + } + } ); + } + catch( csp::TypeError & e ) + { + CSP_THROW( csp::TypeError, + "Unsupported array value type when reading from parquet:" << type->type().asString() ); } } -} +} // namespace namespace csp::python { -//AdapterManager -csp::AdapterManager *create_parquet_input_adapter_manager_impl( PyEngine *engine, const Dictionary &properties, - FileNameGenerator::Ptr fileNameGenerator, - ArrowTableGenerator::Ptr arrowTableGenerator ) +// AdapterManager +csp::AdapterManager * create_parquet_input_adapter_manager_impl( PyEngine * engine, const Dictionary & properties, + FileNameGenerator::Ptr fileNameGenerator, + ArrowTableGenerator::Ptr arrowTableGenerator ) { - auto res = engine -> engine() -> createOwnedObject( properties, fileNameGenerator, arrowTableGenerator ); + auto res = engine->engine()->createOwnedObject( properties, fileNameGenerator, + arrowTableGenerator ); return res; } -static InputAdapter * -create_parquet_input_adapter( csp::AdapterManager *manager, PyEngine *pyengine, PyObject *pyType, PushMode pushMode, - PyObject *args ) +static InputAdapter * create_parquet_input_adapter( csp::AdapterManager * manager, PyEngine * pyengine, + PyObject * pyType, PushMode pushMode, PyObject * args ) { - auto &cspType = pyTypeAsCspType( pyType ); + auto & cspType = pyTypeAsCspType( pyType ); - PyObject *pyProperties; - PyObject *type; + PyObject * pyProperties; + PyObject * type; - auto *parquetManager = dynamic_cast( manager ); + auto * parquetManager = dynamic_cast( manager ); if( !parquetManager ) CSP_THROW( TypeError, "Expected ParquetAdapterManager" ); - if( !PyArg_ParseTuple( args, "O!O!", - &PyType_Type, &type, - &PyDict_Type, &pyProperties ) ) + if( !PyArg_ParseTuple( args, "O!O!", &PyType_Type, &type, &PyDict_Type, &pyProperties ) ) CSP_THROW( PythonPassthrough, "" ); auto propertiesDict = fromPython( pyProperties ); if( propertiesDict.get( "is_array", false ) ) { - auto &&valueType = pyTypeAsCspType( toPythonBorrowed( propertiesDict.get( "array_value_type" ) ) ); - return parquetManager -> getInputAdapter( valueType, propertiesDict, pushMode, - create_numpy_array_reader_impl( valueType ) ); + auto && valueType + = pyTypeAsCspType( toPythonBorrowed( propertiesDict.get( "array_value_type" ) ) ); + return parquetManager->getInputAdapter( valueType, propertiesDict, pushMode, + create_numpy_array_reader_impl( valueType ) ); } else { - return parquetManager -> getInputAdapter( cspType, propertiesDict, pushMode ); + return parquetManager->getInputAdapter( cspType, propertiesDict, pushMode ); } } -static OutputAdapter *create_parquet_output_adapter( csp::AdapterManager *manager, PyEngine *pyengine, PyObject *args ) +static OutputAdapter * create_parquet_output_adapter( csp::AdapterManager * manager, PyEngine * pyengine, + PyObject * args ) { - PyObject *pyProperties; - PyObject *pyType; + PyObject * pyProperties; + PyObject * pyType; - auto *parquetManager = dynamic_cast( manager ); + auto * parquetManager = dynamic_cast( manager ); if( !parquetManager ) CSP_THROW( TypeError, "Expected ParquetAdapterManager" ); - if( !PyArg_ParseTuple( args, "O!O!", - &PyType_Type, &pyType, - &PyDict_Type, &pyProperties ) ) + if( !PyArg_ParseTuple( args, "O!O!", &PyType_Type, &pyType, &PyDict_Type, &pyProperties ) ) CSP_THROW( PythonPassthrough, "" ); - auto &cspType = pyTypeAsCspType( pyType ); - auto &&propertiesDict = fromPython( pyProperties ); + auto & cspType = pyTypeAsCspType( pyType ); + auto && propertiesDict = fromPython( pyProperties ); if( propertiesDict.get( "is_array", false ) ) { - auto &&valueType = pyTypeAsCspType( toPythonBorrowed( propertiesDict.get( "array_value_type" ) ) ); - return parquetManager -> getListOutputAdapter( valueType, propertiesDict, create_numpy_array_writer_impl( valueType ) ); + auto && valueType + = pyTypeAsCspType( toPythonBorrowed( propertiesDict.get( "array_value_type" ) ) ); + return parquetManager->getListOutputAdapter( valueType, propertiesDict, + create_numpy_array_writer_impl( valueType ) ); } else { - return parquetManager -> getOutputAdapter( cspType, propertiesDict ); + return parquetManager->getOutputAdapter( cspType, propertiesDict ); } } -static OutputAdapter *create_parquet_dict_basket_output_adapter( csp::AdapterManager *manager, PyEngine *pyengine, PyObject *args ) +static OutputAdapter * create_parquet_dict_basket_output_adapter( csp::AdapterManager * manager, PyEngine * pyengine, + PyObject * args ) { - PyObject *pyProperties; - PyObject *pyType; - auto *parquetManager = dynamic_cast( manager ); + PyObject * pyProperties; + PyObject * pyType; + auto * parquetManager = dynamic_cast( manager ); if( !parquetManager ) CSP_THROW( TypeError, "Expected ParquetOutputAdapterManager" ); - if( !PyArg_ParseTuple( args, "O!O!", - &PyTuple_Type, &pyType, - &PyDict_Type, &pyProperties ) ) + if( !PyArg_ParseTuple( args, "O!O!", &PyTuple_Type, &pyType, &PyDict_Type, &pyProperties ) ) CSP_THROW( PythonPassthrough, "" ); - PyObject *keyType; - PyObject *valueType; - if( !PyArg_ParseTuple( pyType, "O!O!", - &PyType_Type, &keyType, - &PyType_Type, &valueType ) ) + PyObject * keyType; + PyObject * valueType; + if( !PyArg_ParseTuple( pyType, "O!O!", &PyType_Type, &keyType, &PyType_Type, &valueType ) ) CSP_THROW( PythonPassthrough, "Invalid basket key/value tuple" ); auto cspKeyType = pyTypeAsCspType( keyType ); auto cspValueType = pyTypeAsCspType( valueType ); CSP_THROW( NotImplemented, "Output basket is not implement yet" ); -// PyObject *pyProperties; -// PyObject *pyType; -// -// -// if( !PyArg_ParseTuple( args, "O!O!", -// &PyType_Type, &pyType, -// &PyDict_Type, &pyProperties )) -// CSP_THROW( PythonPassthrough, "" ); -// -// auto &cspType = pyTypeAsCspType( pyType ); -// -// return parquetManager->getOutputAdapter( cspType, fromPython( pyProperties )); + // PyObject *pyProperties; + // PyObject *pyType; + // + // + // if( !PyArg_ParseTuple( args, "O!O!", + // &PyType_Type, &pyType, + // &PyDict_Type, &pyProperties )) + // CSP_THROW( PythonPassthrough, "" ); + // + // auto &cspType = pyTypeAsCspType( pyType ); + // + // return parquetManager->getOutputAdapter( cspType, fromPython( pyProperties )); } -static OutputAdapter *parquet_output_filename_adapter( csp::AdapterManager *manager, PyEngine *pyengine, PyObject *args ) +static OutputAdapter * parquet_output_filename_adapter( csp::AdapterManager * manager, PyEngine * pyengine, + PyObject * args ) { - auto *parquetManager = dynamic_cast( manager ); + auto * parquetManager = dynamic_cast( manager ); if( !parquetManager ) CSP_THROW( TypeError, "Expected ParquetAdapterManager" ); if( !PyArg_ParseTuple( args, "" ) ) CSP_THROW( PythonPassthrough, "" ); - return parquetManager -> createOutputFileNameAdapter(); + return parquetManager->createOutputFileNameAdapter(); } -static PyObject *create_parquet_input_adapter_manager( PyObject *args ) +static PyObject * create_parquet_input_adapter_manager( PyObject * args ) { - CSP_BEGIN_METHOD ; - PyEngine *pyEngine = nullptr; - PyObject *pyProperties = nullptr; - PyObject *pyFileGenerator = nullptr; - - if( !PyArg_ParseTuple( args, "O!O!O!", - &PyEngine::PyType, &pyEngine, - &PyDict_Type, &pyProperties, - &PyFunction_Type, &pyFileGenerator ) ) - CSP_THROW( PythonPassthrough, "" ); - - std::shared_ptr fileNameGenerator; - std::shared_ptr arrowTableGenerator; - - auto dictionary = fromPython( pyProperties ); - if( dictionary.get( "read_from_memory_tables" ) ) - { - arrowTableGenerator = std::make_shared( pyFileGenerator ); - } - else - { - fileNameGenerator = std::make_shared( pyFileGenerator ); - } - auto *adapterMgr = create_parquet_input_adapter_manager_impl( pyEngine, fromPython( pyProperties ), - fileNameGenerator, arrowTableGenerator ); - auto res = PyCapsule_New( adapterMgr, "adapterMgr", nullptr ); - return res; + CSP_BEGIN_METHOD; + PyEngine * pyEngine = nullptr; + PyObject * pyProperties = nullptr; + PyObject * pyFileGenerator = nullptr; + + if( !PyArg_ParseTuple( args, "O!O!O!", &PyEngine::PyType, &pyEngine, &PyDict_Type, &pyProperties, &PyFunction_Type, + &pyFileGenerator ) ) + CSP_THROW( PythonPassthrough, "" ); + + std::shared_ptr fileNameGenerator; + std::shared_ptr arrowTableGenerator; + + auto dictionary = fromPython( pyProperties ); + if( dictionary.get( "read_from_memory_tables" ) ) + { + arrowTableGenerator = std::make_shared( pyFileGenerator ); + } + else + { + fileNameGenerator = std::make_shared( pyFileGenerator ); + } + auto * adapterMgr = create_parquet_input_adapter_manager_impl( pyEngine, fromPython( pyProperties ), + fileNameGenerator, arrowTableGenerator ); + auto res = PyCapsule_New( adapterMgr, "adapterMgr", nullptr ); + return res; CSP_RETURN_NULL; } -//AdapterManager -csp::AdapterManager *create_parquet_output_adapter_manager( PyEngine *engine, const Dictionary &properties ) +// AdapterManager +csp::AdapterManager * create_parquet_output_adapter_manager( PyEngine * engine, const Dictionary & properties ) { ParquetOutputAdapterManager::FileVisitorCallback fileVisitor; DialectGenericType pyFilenameVisitorDG; if( properties.tryGet( "file_visitor", pyFilenameVisitorDG ) ) { PyObjectPtr pyFilenameVisitor = PyObjectPtr::own( toPython( pyFilenameVisitorDG ) ); - fileVisitor = [pyFilenameVisitor]( const std::string & filename ) - { - PyObjectPtr rv = PyObjectPtr::own( PyObject_CallFunction( pyFilenameVisitor.get(), "O", PyObjectPtr::own( toPython( filename ) ).get() ) ); - if( !rv.get() ) - CSP_THROW( PythonPassthrough, "" ); - }; + fileVisitor = [pyFilenameVisitor]( const std::string & filename ) + { + PyObjectPtr rv = PyObjectPtr::own( + PyObject_CallFunction( pyFilenameVisitor.get(), "O", PyObjectPtr::own( toPython( filename ) ).get() ) ); + if( !rv.get() ) + CSP_THROW( PythonPassthrough, "" ); + }; } - return engine -> engine() -> createOwnedObject( properties, fileVisitor ); + return engine->engine()->createOwnedObject( properties, fileVisitor ); } - REGISTER_ADAPTER_MANAGER_CUSTOM_CREATOR( _parquet_input_adapter_manager, create_parquet_input_adapter_manager ); REGISTER_ADAPTER_MANAGER( _parquet_output_adapter_manager, create_parquet_output_adapter_manager ); @@ -675,16 +692,11 @@ REGISTER_OUTPUT_ADAPTER( _parquet_dict_basket_output_adapter, create_parquet_dic REGISTER_OUTPUT_ADAPTER( _parquet_output_filename_adapter, parquet_output_filename_adapter ); static PyModuleDef _parquetadapterimpl_module = { - PyModuleDef_HEAD_INIT, - "_parquetadapterimpl", - "_parquetadapterimpl c++ module", - -1, - NULL, NULL, NULL, NULL, NULL -}; + PyModuleDef_HEAD_INIT, "_parquetadapterimpl", "_parquetadapterimpl c++ module", -1, NULL, NULL, NULL, NULL, NULL }; PyMODINIT_FUNC PyInit__parquetadapterimpl( void ) { - PyObject *m; + PyObject * m; m = PyModule_Create( &_parquetadapterimpl_module ); if( m == NULL ) @@ -702,4 +714,4 @@ PyMODINIT_FUNC PyInit__parquetadapterimpl( void ) return m; } -} +} // namespace csp::python diff --git a/cpp/csp/python/adapters/websocketadapterimpl.cpp b/cpp/csp/python/adapters/websocketadapterimpl.cpp index d636932d..0a8fdd37 100644 --- a/cpp/csp/python/adapters/websocketadapterimpl.cpp +++ b/cpp/csp/python/adapters/websocketadapterimpl.cpp @@ -16,64 +16,67 @@ namespace csp::python using namespace csp; using namespace csp::python; -//AdapterManager +// AdapterManager csp::AdapterManager * create_websocket_adapter_manager( PyEngine * engine, const Dictionary & properties ) { - return engine -> engine() -> createOwnedObject( properties ); + return engine->engine()->createOwnedObject( properties ); } -static InputAdapter * create_websocket_input_adapter( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * pyType, PushMode pushMode, PyObject * args ) +static InputAdapter * create_websocket_input_adapter( csp::AdapterManager * manager, PyEngine * pyengine, + PyObject * pyType, PushMode pushMode, PyObject * args ) { auto & cspType = pyTypeAsCspType( pyType ); PyObject * pyProperties; PyObject * type; - auto * websocketManager = dynamic_cast( manager ); + auto * websocketManager = dynamic_cast( manager ); if( !websocketManager ) CSP_THROW( TypeError, "Expected WebsocketClientAdapterManager" ); - if( !PyArg_ParseTuple( args, "O!O!", - &PyType_Type, &type, - &PyDict_Type, &pyProperties ) ) + if( !PyArg_ParseTuple( args, "O!O!", &PyType_Type, &type, &PyDict_Type, &pyProperties ) ) CSP_THROW( PythonPassthrough, "" ); - return websocketManager -> getInputAdapter( cspType, pushMode, fromPython( pyProperties ) ); + return websocketManager->getInputAdapter( cspType, pushMode, fromPython( pyProperties ) ); } -static OutputAdapter * create_websocket_output_adapter( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * args ) +static OutputAdapter * create_websocket_output_adapter( csp::AdapterManager * manager, PyEngine * pyengine, + PyObject * args ) { - auto * websocketManager = dynamic_cast( manager ); + auto * websocketManager = dynamic_cast( manager ); if( !websocketManager ) CSP_THROW( TypeError, "Expected WebsocketClientAdapterManager" ); - return websocketManager -> getOutputAdapter(); + return websocketManager->getOutputAdapter(); } -static OutputAdapter * create_websocket_header_update_adapter( csp::AdapterManager * manager, PyEngine * pyengine, PyObject * args ) +static OutputAdapter * create_websocket_header_update_adapter( csp::AdapterManager * manager, PyEngine * pyengine, + PyObject * args ) { - auto * websocketManager = dynamic_cast( manager ); + auto * websocketManager = dynamic_cast( manager ); if( !websocketManager ) CSP_THROW( TypeError, "Expected WebsocketClientAdapterManager" ); - return websocketManager -> getHeaderUpdateAdapter(); + return websocketManager->getHeaderUpdateAdapter(); } REGISTER_ADAPTER_MANAGER( _websocket_adapter_manager, create_websocket_adapter_manager ); -REGISTER_INPUT_ADAPTER( _websocket_input_adapter, create_websocket_input_adapter ); -REGISTER_OUTPUT_ADAPTER( _websocket_output_adapter, create_websocket_output_adapter ); -REGISTER_OUTPUT_ADAPTER( _websocket_header_update_adapter, create_websocket_header_update_adapter); - -static PyModuleDef _websocketadapterimpl_module = { - PyModuleDef_HEAD_INIT, - "_websocketadapterimpl", - "_websocketadapterimpl c++ module", - -1, - NULL, NULL, NULL, NULL, NULL -}; - -PyMODINIT_FUNC PyInit__websocketadapterimpl(void) +REGISTER_INPUT_ADAPTER( _websocket_input_adapter, create_websocket_input_adapter ); +REGISTER_OUTPUT_ADAPTER( _websocket_output_adapter, create_websocket_output_adapter ); +REGISTER_OUTPUT_ADAPTER( _websocket_header_update_adapter, create_websocket_header_update_adapter ); + +static PyModuleDef _websocketadapterimpl_module = { PyModuleDef_HEAD_INIT, + "_websocketadapterimpl", + "_websocketadapterimpl c++ module", + -1, + NULL, + NULL, + NULL, + NULL, + NULL }; + +PyMODINIT_FUNC PyInit__websocketadapterimpl( void ) { - PyObject* m; + PyObject * m; - m = PyModule_Create( &_websocketadapterimpl_module); + m = PyModule_Create( &_websocketadapterimpl_module ); if( m == NULL ) return NULL; @@ -83,4 +86,4 @@ PyMODINIT_FUNC PyInit__websocketadapterimpl(void) return m; } -} +} // namespace csp::python diff --git a/cpp/csp/python/cspbaselibimpl.cpp b/cpp/csp/python/cspbaselibimpl.cpp index 1ada5ced..80e51a55 100644 --- a/cpp/csp/python/cspbaselibimpl.cpp +++ b/cpp/csp/python/cspbaselibimpl.cpp @@ -1,7 +1,7 @@ -#include -#include #include +#include #include +#include #include #include @@ -20,35 +20,32 @@ DECLARE_CPPNODE( exprtk_impl ) class BaseValueContainer { public: - virtual ~BaseValueContainer() = default; - virtual void setValue( const TimeSeriesProvider * ) = 0; - virtual bool registerValue( exprtk::symbol_table &expr, const std::string &variableName ) = 0; + virtual ~BaseValueContainer() = default; + virtual void setValue( const TimeSeriesProvider * ) = 0; + virtual bool registerValue( exprtk::symbol_table & expr, const std::string & variableName ) = 0; }; - template< typename T > + template class ValueContainer : public BaseValueContainer { public: - void setValue( const TimeSeriesProvider *tsProvider ) override - { - m_value = tsProvider -> lastValueTyped(); - } + void setValue( const TimeSeriesProvider * tsProvider ) override { m_value = tsProvider->lastValueTyped(); } - bool registerValue( exprtk::symbol_table &expr, const std::string &variableName ) override + bool registerValue( exprtk::symbol_table & expr, const std::string & variableName ) override { registerValueImpl( expr, variableName ); return true; } private: - template , bool> = true> - void registerValueImpl( exprtk::symbol_table &symbolTable, const std::string &variableName ) + template, bool> = true> + void registerValueImpl( exprtk::symbol_table & symbolTable, const std::string & variableName ) { symbolTable.add_variable( variableName, m_value ); } - template , bool> = true> - void registerValueImpl( exprtk::symbol_table &symbolTable, const std::string &variableName ) + template, bool> = true> + void registerValueImpl( exprtk::symbol_table & symbolTable, const std::string & variableName ) { symbolTable.add_stringvar( variableName, m_value ); } @@ -59,13 +56,17 @@ DECLARE_CPPNODE( exprtk_impl ) class NumpyArrayValueContainer : public BaseValueContainer { public: - NumpyArrayValueContainer() : m_arr_size(-1) {} + NumpyArrayValueContainer() + : m_arr_size( -1 ) + { + } - void validateArray( PyArrayObject* arr ) + void validateArray( PyArrayObject * arr ) { auto dim = PyArray_NDIM( arr ); if( dim != 1 ) - CSP_THROW( ValueError, "csp.exprtk recieved an array of dim " << dim << " but can only take 1D arrays" ); + CSP_THROW( ValueError, + "csp.exprtk recieved an array of dim " << dim << " but can only take 1D arrays" ); if( !PyArray_CHKFLAGS( arr, NPY_ARRAY_OWNDATA ) ) CSP_THROW( ValueError, "csp.exprtk requires arrays be naturally strided" ); @@ -74,42 +75,44 @@ DECLARE_CPPNODE( exprtk_impl ) CSP_THROW( ValueError, "csp.exprtk requires arrays to contain floats" ); } - void setValue( const TimeSeriesProvider *tsProvider ) override + void setValue( const TimeSeriesProvider * tsProvider ) override { - PyArrayObject* arr = (PyArrayObject*) csp::python::toPythonBorrowed(tsProvider -> lastValueTyped()); + PyArrayObject * arr + = (PyArrayObject *)csp::python::toPythonBorrowed( tsProvider->lastValueTyped() ); // register on first tick if( m_arr_size == -1 ) { validateArray( arr ); - m_arr_size = PyArray_SIZE( arr ); - double* data = reinterpret_cast( PyArray_DATA( arr ) ); - m_view = std::make_unique>( data, m_arr_size ); + m_arr_size = PyArray_SIZE( arr ); + double * data = reinterpret_cast( PyArray_DATA( arr ) ); + m_view = std::make_unique>( data, m_arr_size ); - m_symbolTable -> add_vector( m_var_name, *m_view ); + m_symbolTable->add_vector( m_var_name, *m_view ); } else { if( PyArray_SIZE( arr ) != m_arr_size ) - CSP_THROW( ValueError, "csp.exprtk NumPy array input must have same size each tick, but first saw " << m_arr_size - << " and now saw " << PyArray_SIZE( arr ) << " for " << m_var_name ); + CSP_THROW( ValueError, + "csp.exprtk NumPy array input must have same size each tick, but first saw " + << m_arr_size << " and now saw " << PyArray_SIZE( arr ) << " for " << m_var_name ); validateArray( arr ); - double* data = reinterpret_cast( PyArray_DATA( arr ) ); - m_view -> rebase( data ); + double * data = reinterpret_cast( PyArray_DATA( arr ) ); + m_view->rebase( data ); } } - bool registerValue( exprtk::symbol_table &symbolTable, const std::string &variableName ) override + bool registerValue( exprtk::symbol_table & symbolTable, const std::string & variableName ) override { // store symbol table and var name so we can use them to register in setValue, on first tick m_symbolTable = &symbolTable; - m_var_name = variableName; + m_var_name = variableName; return false; } private: - exprtk::symbol_table *m_symbolTable; + exprtk::symbol_table * m_symbolTable; std::string m_var_name; int64_t m_arr_size; std::unique_ptr> m_view; @@ -118,20 +121,24 @@ DECLARE_CPPNODE( exprtk_impl ) struct csp_now_fn : public exprtk::ifunction { public: - csp_now_fn() : exprtk::ifunction(0) {} - double operator()() { return ( m_engine -> rootEngine() -> now() ).asNanoseconds() / 1e9; } + csp_now_fn() + : exprtk::ifunction( 0 ) + { + } + double operator()() { return ( m_engine->rootEngine()->now() ).asNanoseconds() / 1e9; } void setEngine( csp::Engine * engine ) { m_engine = engine; } + private: csp::Engine * m_engine; }; - SCALAR_INPUT( std::string, expression_str ); - TS_DICTBASKET_INPUT( DialectGenericType, inputs ); - SCALAR_INPUT( DictionaryPtr, state_vars ); - SCALAR_INPUT( DictionaryPtr, constants ); - SCALAR_INPUT( DictionaryPtr, functions ); - TS_INPUT( Generic, trigger ); - SCALAR_INPUT( bool, use_trigger ); + SCALAR_INPUT( std::string, expression_str ); + TS_DICTBASKET_INPUT( DialectGenericType, inputs ); + SCALAR_INPUT( DictionaryPtr, state_vars ); + SCALAR_INPUT( DictionaryPtr, constants ); + SCALAR_INPUT( DictionaryPtr, functions ); + TS_INPUT( Generic, trigger ); + SCALAR_INPUT( bool, use_trigger ); TS_OUTPUT( Generic ); STATE_VAR( exprtk::function_compositor, s_compositor ); @@ -146,7 +153,9 @@ DECLARE_CPPNODE( exprtk_impl ) s_expr.register_symbol_table( s_compositor.symbol_table() ); if( !s_parser.compile( expression_str, s_expr ) ) - CSP_THROW( ValueError, "cannot compile expression: " << std::string( expression_str ) << " ERROR: " << s_parser.error() ); + CSP_THROW( ValueError, + "cannot compile expression: " << std::string( expression_str ) + << " ERROR: " << s_parser.error() ); s_isCompiled = true; } @@ -155,69 +164,72 @@ DECLARE_CPPNODE( exprtk_impl ) START() { - s_isCompiled = false; - bool all_registered = true; - exprtk::symbol_table& symbolTable = s_compositor.symbol_table(); + s_isCompiled = false; + bool all_registered = true; + exprtk::symbol_table & symbolTable = s_compositor.symbol_table(); for( size_t elem = 0; elem < inputs.size(); ++elem ) { - auto &&inputName = inputs.shape()[ elem ]; - auto typ = inputs[ elem ].type(); + auto && inputName = inputs.shape()[elem]; + auto typ = inputs[elem].type(); - if( typ -> type() == CspType::Type::DIALECT_GENERIC ) + if( typ->type() == CspType::Type::DIALECT_GENERIC ) { s_valuesContainer.push_back( std::make_unique() ); } else { PartialSwitchCspType::invoke( - typ, - [ this ]( auto tag ) - { - s_valuesContainer.push_back( std::make_unique>() ); - } ); + typ, + [this]( auto tag ) { + s_valuesContainer.push_back( + std::make_unique>() ); + } ); } - all_registered &= s_valuesContainer.back() -> registerValue( symbolTable, inputName ); + all_registered &= s_valuesContainer.back()->registerValue( symbolTable, inputName ); } - for( auto it = state_vars.value() -> begin(); it != state_vars.value() -> end(); ++it ) + for( auto it = state_vars.value()->begin(); it != state_vars.value()->end(); ++it ) { if( it.hasValue() ) { symbolTable.create_stringvar( it.key() ); - symbolTable.get_stringvar( it.key() ) -> ref() = it.value(); + symbolTable.get_stringvar( it.key() )->ref() = it.value(); } else if( it.hasValue() || it.hasValue() ) { - symbolTable.create_variable(it.key()); - symbolTable.get_variable( it.key() ) -> ref() = it.value(); + symbolTable.create_variable( it.key() ); + symbolTable.get_variable( it.key() )->ref() = it.value(); } else - CSP_THROW( ValueError, "state_vars dictionary contains " << it.key() << " with unsupported type (need be string or float)" ); + CSP_THROW( ValueError, + "state_vars dictionary contains " << it.key() + << " with unsupported type (need be string or float)" ); } - for( auto it = constants.value() -> begin(); it != constants.value() -> end(); ++it ) + for( auto it = constants.value()->begin(); it != constants.value()->end(); ++it ) { if( it.hasValue() || it.hasValue() ) symbolTable.add_constant( it.key(), it.value() ); else - CSP_THROW( ValueError, "constants dictionary contains " << it.key() << " with unsupported type (need be float)" ); + CSP_THROW( ValueError, + "constants dictionary contains " << it.key() << " with unsupported type (need be float)" ); } - if( functions.value() -> size() > 0 ) + if( functions.value()->size() > 0 ) { typedef exprtk::function_compositor compositor_t; typedef typename compositor_t::function function_t; - for( auto it = functions.value() -> begin(); it != functions.value() -> end(); ++it ) + for( auto it = functions.value()->begin(); it != functions.value()->end(); ++it ) { - csp::python::PyObjectPtr fnInfo = csp::python::PyObjectPtr::own( csp::python::toPython( it.value() ) ); + csp::python::PyObjectPtr fnInfo + = csp::python::PyObjectPtr::own( csp::python::toPython( it.value() ) ); const char * body; PyObject * vars; - - if( !PyArg_ParseTuple( fnInfo.get(), "O!s", &PyTuple_Type, &vars , &body ) ) + if( !PyArg_ParseTuple( fnInfo.get(), "O!s", &PyTuple_Type, &vars, &body ) ) { CSP_THROW( csp::python::PythonPassthrough, "could not parse function info in csp.exprtk" ); } @@ -258,7 +270,8 @@ DECLARE_CPPNODE( exprtk_impl ) s_compositor.add( function_t( it.key(), body, arg1, arg2, arg3, arg4 ) ); break; default: - CSP_THROW( ValueError, "csp.exprtk given too many variables (" << numVars << "), max supported is 4" ); + CSP_THROW( ValueError, + "csp.exprtk given too many variables (" << numVars << "), max supported is 4" ); } } } @@ -277,16 +290,16 @@ DECLARE_CPPNODE( exprtk_impl ) { if( use_trigger ) { - for( auto &&inputIt = inputs.validinputs(); inputIt; ++inputIt ) + for( auto && inputIt = inputs.validinputs(); inputIt; ++inputIt ) { - s_valuesContainer[ inputIt.elemId() ] -> setValue( inputIt.get() ); + s_valuesContainer[inputIt.elemId()]->setValue( inputIt.get() ); } } else { - for( auto &&inputIt = inputs.tickedinputs(); inputIt; ++inputIt ) + for( auto && inputIt = inputs.tickedinputs(); inputIt; ++inputIt ) { - s_valuesContainer[ inputIt.elemId() ] -> setValue( inputIt.get() ); + s_valuesContainer[inputIt.elemId()]->setValue( inputIt.get() ); } } @@ -295,37 +308,36 @@ DECLARE_CPPNODE( exprtk_impl ) if( unlikely( !s_isCompiled ) ) compile_expression(); - const CspType* outputType = unnamed_output().type(); + const CspType * outputType = unnamed_output().type(); if( outputType->type() == CspType::Type::DOUBLE ) { RETURN( s_expr.value() ); } else { - s_expr.value(); // need this to get the expression to evaluate + s_expr.value(); // need this to get the expression to evaluate - const exprtk::results_context& results = s_expr.results(); - npy_intp numResults = results.count(); - PyObject* out = PyArray_EMPTY(1, &numResults, NPY_DOUBLE, 0 ); // 1D array - double* data = reinterpret_cast( PyArray_DATA( ( PyArrayObject* )out ) ); + const exprtk::results_context & results = s_expr.results(); + npy_intp numResults = results.count(); + PyObject * out = PyArray_EMPTY( 1, &numResults, NPY_DOUBLE, 0 ); // 1D array + double * data = reinterpret_cast( PyArray_DATA( (PyArrayObject *)out ) ); typedef exprtk::results_context::type_store_t::scalar_view scalar_t; - for (npy_intp i = 0; i < numResults; ++i) + for( npy_intp i = 0; i < numResults; ++i ) { - data[i] = scalar_t(results[i])(); + data[i] = scalar_t( results[i] )(); } RETURN( csp::python::PyObjectPtr::own( out ) ); } } } - }; EXPORT_CPPNODE( exprtk_impl ); -} +} // namespace csp::cppnodes // Base nodes REGISTER_CPPNODE( csp::cppnodes, sample ); @@ -351,19 +363,14 @@ REGISTER_CPPNODE( csp::cppnodes, struct_collectts ); REGISTER_CPPNODE( csp::cppnodes, exprtk_impl ); -static PyModuleDef _cspbaselibimpl_module = { - PyModuleDef_HEAD_INIT, - "_cspbaselibimpl", - "_cspbaselibimpl c++ module", - -1, - NULL, NULL, NULL, NULL, NULL -}; +static PyModuleDef _cspbaselibimpl_module + = { PyModuleDef_HEAD_INIT, "_cspbaselibimpl", "_cspbaselibimpl c++ module", -1, NULL, NULL, NULL, NULL, NULL }; -PyMODINIT_FUNC PyInit__cspbaselibimpl(void) +PyMODINIT_FUNC PyInit__cspbaselibimpl( void ) { - PyObject* m; + PyObject * m; - m = PyModule_Create( &_cspbaselibimpl_module); + m = PyModule_Create( &_cspbaselibimpl_module ); if( m == NULL ) return NULL; diff --git a/cpp/csp/python/cspbasketlibimpl.cpp b/cpp/csp/python/cspbasketlibimpl.cpp index c2c59c61..bbe414ca 100644 --- a/cpp/csp/python/cspbasketlibimpl.cpp +++ b/cpp/csp/python/cspbasketlibimpl.cpp @@ -4,19 +4,14 @@ REGISTER_CPPNODE( csp::cppnodes, _sync_list ); REGISTER_CPPNODE( csp::cppnodes, _sample_list ); -static PyModuleDef _cspbasketlibimpl_module = { - PyModuleDef_HEAD_INIT, - "_cspbasketlibimpl", - "_cspbasketlibimpl c++ module", - -1, - NULL, NULL, NULL, NULL, NULL -}; +static PyModuleDef _cspbasketlibimpl_module + = { PyModuleDef_HEAD_INIT, "_cspbasketlibimpl", "_cspbasketlibimpl c++ module", -1, NULL, NULL, NULL, NULL, NULL }; -PyMODINIT_FUNC PyInit__cspbasketlibimpl(void) +PyMODINIT_FUNC PyInit__cspbasketlibimpl( void ) { - PyObject* m; + PyObject * m; - m = PyModule_Create( &_cspbasketlibimpl_module); + m = PyModule_Create( &_cspbasketlibimpl_module ); if( m == NULL ) return NULL; diff --git a/cpp/csp/python/cspimpl.cpp b/cpp/csp/python/cspimpl.cpp index 8527aeb6..31a3e20b 100644 --- a/cpp/csp/python/cspimpl.cpp +++ b/cpp/csp/python/cspimpl.cpp @@ -8,77 +8,74 @@ namespace csp::python { -static PyObject * _csp_now( PyObject*, PyObject * nodeptr ) +static PyObject * _csp_now( PyObject *, PyObject * nodeptr ) { CSP_BEGIN_METHOD; csp::Node * node = reinterpret_cast( fromPython( nodeptr ) ); - return toPython( node -> now() ); + return toPython( node->now() ); CSP_RETURN_NULL; } -static PyObject * _engine_start_time( PyObject*, PyObject * nodeptr ) +static PyObject * _engine_start_time( PyObject *, PyObject * nodeptr ) { CSP_BEGIN_METHOD; csp::Node * node = reinterpret_cast( fromPython( nodeptr ) ); - return toPython( node -> rootEngine() -> startTime() ); + return toPython( node->rootEngine()->startTime() ); CSP_RETURN_NULL; } -static PyObject * _engine_stats( PyObject*, PyObject * nodeptr ) +static PyObject * _engine_stats( PyObject *, PyObject * nodeptr ) { CSP_BEGIN_METHOD; csp::Node * node = reinterpret_cast( fromPython( nodeptr ) ); - return toPython( node -> rootEngine() -> engine_stats() ); + return toPython( node->rootEngine()->engine_stats() ); CSP_RETURN_NULL; } - -static PyObject * _engine_end_time( PyObject*, PyObject * nodeptr ) +static PyObject * _engine_end_time( PyObject *, PyObject * nodeptr ) { CSP_BEGIN_METHOD; csp::Node * node = reinterpret_cast( fromPython( nodeptr ) ); - return toPython( node -> rootEngine() -> endTime() ); + return toPython( node->rootEngine()->endTime() ); CSP_RETURN_NULL; } -static PyObject * _csp_stop_engine( PyObject*, PyObject * args, PyObject * kwargs ) +static PyObject * _csp_stop_engine( PyObject *, PyObject * args, PyObject * kwargs ) { CSP_BEGIN_METHOD; int dynamicOnly = 0; uint64_t nodeptr; static const char * kwlist[] = { "node", "dynamic", nullptr }; - if( !PyArg_ParseTupleAndKeywords( args, kwargs, "L|p", ( char ** ) kwlist, &nodeptr, &dynamicOnly ) ) + if( !PyArg_ParseTupleAndKeywords( args, kwargs, "L|p", (char **)kwlist, &nodeptr, &dynamicOnly ) ) CSP_THROW( PythonPassthrough, "" ); csp::Node * node = reinterpret_cast( nodeptr ); - if( dynamicOnly && !node -> engine() -> isRootEngine() ) - static_cast( node -> engine() ) -> shutdown(); + if( dynamicOnly && !node->engine()->isRootEngine() ) + static_cast( node->engine() )->shutdown(); else - node -> rootEngine() -> shutdown(); + node->rootEngine()->shutdown(); CSP_RETURN_NONE; } -//Ingloriously stolen from Python 3.7.0! -static PyObject *_create_traceback( PyObject *, PyObject * args ) +// Ingloriously stolen from Python 3.7.0! +static PyObject * _create_traceback( PyObject *, PyObject * args ) { CSP_BEGIN_METHOD; - PyObject *next; - PyFrameObject *frame; + PyObject * next; + PyFrameObject * frame; int lasti; int lineno; - if( !PyArg_ParseTuple( args, "OO!ii", - &next, - &PyFrame_Type, &frame, &lasti, &lineno ) ) + if( !PyArg_ParseTuple( args, "OO!ii", &next, &PyFrame_Type, &frame, &lasti, &lineno ) ) CSP_THROW( PythonPassthrough, "" ); if( next == Py_None ) @@ -86,16 +83,16 @@ static PyObject *_create_traceback( PyObject *, PyObject * args ) else if( !PyTraceBack_Check( next ) ) CSP_THROW( TypeError, "expected traceback type" ); - PyTracebackObject *tb; + PyTracebackObject * tb; tb = PyObject_GC_New( PyTracebackObject, &PyTraceBack_Type ); if( tb != NULL ) { - Py_XINCREF(next); - tb -> tb_next = (PyTracebackObject * ) next; - Py_XINCREF(frame); - tb -> tb_frame = frame; - tb -> tb_lasti = lasti; - tb -> tb_lineno = lineno; + Py_XINCREF( next ); + tb->tb_next = (PyTracebackObject *)next; + Py_XINCREF( frame ); + tb->tb_frame = frame; + tb->tb_lasti = lasti; + tb->tb_lineno = lineno; PyObject_GC_Track( tb ); } @@ -104,7 +101,7 @@ static PyObject *_create_traceback( PyObject *, PyObject * args ) CSP_RETURN_NULL } -static PyObject *_set_capture_cpp_backtrace( PyObject *, PyObject *args ) +static PyObject * _set_capture_cpp_backtrace( PyObject *, PyObject * args ) { CSP_BEGIN_METHOD; int value; @@ -115,31 +112,24 @@ static PyObject *_set_capture_cpp_backtrace( PyObject *, PyObject *args ) CSP_RETURN_NONE; } +static PyMethodDef _cspimpl_methods[] + = { { "_csp_now", (PyCFunction)_csp_now, METH_O, "current engine time" }, + { "_csp_engine_start_time", (PyCFunction)_engine_start_time, METH_O, "engine start time" }, + { "_csp_engine_end_time", (PyCFunction)_engine_end_time, METH_O, "engine end time" }, + { "_csp_stop_engine", (PyCFunction)_csp_stop_engine, METH_VARARGS | METH_KEYWORDS, "stop engine" }, + { "create_traceback", (PyCFunction)_create_traceback, METH_VARARGS, "internal" }, + { "_csp_engine_stats", (PyCFunction)_engine_stats, METH_O, "engine statistics" }, + { "set_capture_cpp_backtrace", (PyCFunction)_set_capture_cpp_backtrace, METH_VARARGS, "internal" }, + { nullptr } }; + +static PyModuleDef _cspimpl_module + = { PyModuleDef_HEAD_INIT, "_cspimpl", "_cspimpl c++ module", -1, _cspimpl_methods, NULL, NULL, NULL, NULL }; -static PyMethodDef _cspimpl_methods[] = { - {"_csp_now", (PyCFunction) _csp_now, METH_O, "current engine time"}, - {"_csp_engine_start_time", (PyCFunction) _engine_start_time, METH_O, "engine start time"}, - {"_csp_engine_end_time", (PyCFunction) _engine_end_time, METH_O, "engine end time"}, - {"_csp_stop_engine", (PyCFunction) _csp_stop_engine, METH_VARARGS | METH_KEYWORDS, "stop engine"}, - {"create_traceback", (PyCFunction) _create_traceback, METH_VARARGS, "internal"}, - {"_csp_engine_stats", (PyCFunction) _engine_stats, METH_O, "engine statistics"}, - {"set_capture_cpp_backtrace", (PyCFunction) _set_capture_cpp_backtrace, METH_VARARGS, "internal"}, - {nullptr} -}; - -static PyModuleDef _cspimpl_module = { - PyModuleDef_HEAD_INIT, - "_cspimpl", - "_cspimpl c++ module", - -1, - _cspimpl_methods, NULL, NULL, NULL, NULL -}; - -PyMODINIT_FUNC PyInit__cspimpl(void) +PyMODINIT_FUNC PyInit__cspimpl( void ) { - PyObject* m; + PyObject * m; - m = PyModule_Create( &_cspimpl_module); + m = PyModule_Create( &_cspimpl_module ); if( m == NULL ) return NULL; @@ -149,4 +139,4 @@ PyMODINIT_FUNC PyInit__cspimpl(void) return m; } -} +} // namespace csp::python diff --git a/cpp/csp/python/cspmathimpl.cpp b/cpp/csp/python/cspmathimpl.cpp index 27559047..94b54837 100644 --- a/cpp/csp/python/cspmathimpl.cpp +++ b/cpp/csp/python/cspmathimpl.cpp @@ -1,6 +1,6 @@ -#include #include #include +#include // Math ops REGISTER_CPPNODE( csp::cppnodes, add_f ); @@ -75,19 +75,14 @@ REGISTER_CPPNODE( csp::cppnodes, ge_i ); REGISTER_CPPNODE( csp::cppnodes, le_f ); REGISTER_CPPNODE( csp::cppnodes, le_i ); -static PyModuleDef _cspmathimpl_module = { - PyModuleDef_HEAD_INIT, - "_cspmathimpl", - "_cspmathimpl c++ module", - -1, - NULL, NULL, NULL, NULL, NULL -}; +static PyModuleDef _cspmathimpl_module + = { PyModuleDef_HEAD_INIT, "_cspmathimpl", "_cspmathimpl c++ module", -1, NULL, NULL, NULL, NULL, NULL }; -PyMODINIT_FUNC PyInit__cspmathimpl(void) +PyMODINIT_FUNC PyInit__cspmathimpl( void ) { - PyObject* m; + PyObject * m; - m = PyModule_Create( &_cspmathimpl_module); + m = PyModule_Create( &_cspmathimpl_module ); if( m == NULL ) return NULL; diff --git a/cpp/csp/python/cspnpstatsimpl.cpp b/cpp/csp/python/cspnpstatsimpl.cpp index ea0072a2..66e164c8 100644 --- a/cpp/csp/python/cspnpstatsimpl.cpp +++ b/cpp/csp/python/cspnpstatsimpl.cpp @@ -4,67 +4,61 @@ // Register nodes and create module // Data processing -REGISTER_CPPNODE( csp::python, _np_tick_window_updates ); -REGISTER_CPPNODE( csp::python, _np_time_window_updates ); -REGISTER_CPPNODE( csp::python, _cross_sectional_as_np ); -REGISTER_CPPNODE( csp::python, _np_cross_sectional_as_list ); -REGISTER_CPPNODE( csp::python, _np_cross_sectional_as_np ); -REGISTER_CPPNODE( csp::python, _list_to_np ); -REGISTER_CPPNODE( csp::python, _np_to_list ); -REGISTER_CPPNODE( csp::python, _sync_nan_np ); +REGISTER_CPPNODE( csp::python, _np_tick_window_updates ); +REGISTER_CPPNODE( csp::python, _np_time_window_updates ); +REGISTER_CPPNODE( csp::python, _cross_sectional_as_np ); +REGISTER_CPPNODE( csp::python, _np_cross_sectional_as_list ); +REGISTER_CPPNODE( csp::python, _np_cross_sectional_as_np ); +REGISTER_CPPNODE( csp::python, _list_to_np ); +REGISTER_CPPNODE( csp::python, _np_to_list ); +REGISTER_CPPNODE( csp::python, _sync_nan_np ); // Computation nodes -REGISTER_CPPNODE( csp::python, _np_count ); -REGISTER_CPPNODE( csp::python, _np_first ); -REGISTER_CPPNODE( csp::python, _np_last ); -REGISTER_CPPNODE( csp::python, _np_sum ); -REGISTER_CPPNODE( csp::python, _np_kahan_sum ); -REGISTER_CPPNODE( csp::python, _np_mean ); -REGISTER_CPPNODE( csp::python, _np_prod ); -REGISTER_CPPNODE( csp::python, _np_unique ); -REGISTER_CPPNODE( csp::python, _np_quantile ); -REGISTER_CPPNODE( csp::python, _np_min_max ); -REGISTER_CPPNODE( csp::python, _np_rank ); -REGISTER_CPPNODE( csp::python, _np_arg_min_max ); -REGISTER_CPPNODE( csp::python, _np_weighted_mean ); -REGISTER_CPPNODE( csp::python, _np_var ); -REGISTER_CPPNODE( csp::python, _np_weighted_var ); -REGISTER_CPPNODE( csp::python, _np_sem ); -REGISTER_CPPNODE( csp::python, _np_weighted_sem ); -REGISTER_CPPNODE( csp::python, _np_covar ); -REGISTER_CPPNODE( csp::python, _np_weighted_covar ); -REGISTER_CPPNODE( csp::python, _np_corr ); -REGISTER_CPPNODE( csp::python, _np_weighted_corr ); -REGISTER_CPPNODE( csp::python, _np_cov_matrix ); -REGISTER_CPPNODE( csp::python, _np_weighted_cov_matrix ); -REGISTER_CPPNODE( csp::python, _np_corr_matrix ); -REGISTER_CPPNODE( csp::python, _np_weighted_corr_matrix ); -REGISTER_CPPNODE( csp::python, _np_skew ); -REGISTER_CPPNODE( csp::python, _np_weighted_skew ); -REGISTER_CPPNODE( csp::python, _np_kurt ); -REGISTER_CPPNODE( csp::python, _np_weighted_kurt ); +REGISTER_CPPNODE( csp::python, _np_count ); +REGISTER_CPPNODE( csp::python, _np_first ); +REGISTER_CPPNODE( csp::python, _np_last ); +REGISTER_CPPNODE( csp::python, _np_sum ); +REGISTER_CPPNODE( csp::python, _np_kahan_sum ); +REGISTER_CPPNODE( csp::python, _np_mean ); +REGISTER_CPPNODE( csp::python, _np_prod ); +REGISTER_CPPNODE( csp::python, _np_unique ); +REGISTER_CPPNODE( csp::python, _np_quantile ); +REGISTER_CPPNODE( csp::python, _np_min_max ); +REGISTER_CPPNODE( csp::python, _np_rank ); +REGISTER_CPPNODE( csp::python, _np_arg_min_max ); +REGISTER_CPPNODE( csp::python, _np_weighted_mean ); +REGISTER_CPPNODE( csp::python, _np_var ); +REGISTER_CPPNODE( csp::python, _np_weighted_var ); +REGISTER_CPPNODE( csp::python, _np_sem ); +REGISTER_CPPNODE( csp::python, _np_weighted_sem ); +REGISTER_CPPNODE( csp::python, _np_covar ); +REGISTER_CPPNODE( csp::python, _np_weighted_covar ); +REGISTER_CPPNODE( csp::python, _np_corr ); +REGISTER_CPPNODE( csp::python, _np_weighted_corr ); +REGISTER_CPPNODE( csp::python, _np_cov_matrix ); +REGISTER_CPPNODE( csp::python, _np_weighted_cov_matrix ); +REGISTER_CPPNODE( csp::python, _np_corr_matrix ); +REGISTER_CPPNODE( csp::python, _np_weighted_corr_matrix ); +REGISTER_CPPNODE( csp::python, _np_skew ); +REGISTER_CPPNODE( csp::python, _np_weighted_skew ); +REGISTER_CPPNODE( csp::python, _np_kurt ); +REGISTER_CPPNODE( csp::python, _np_weighted_kurt ); // EMA nodes -REGISTER_CPPNODE( csp::python, _np_ema_compute ); -REGISTER_CPPNODE( csp::python, _np_ema_adjusted ); -REGISTER_CPPNODE( csp::python, _np_ema_timewise ); -REGISTER_CPPNODE( csp::python, _np_ema_debias_alpha ); -REGISTER_CPPNODE( csp::python, _np_ema_debias_halflife ); +REGISTER_CPPNODE( csp::python, _np_ema_compute ); +REGISTER_CPPNODE( csp::python, _np_ema_adjusted ); +REGISTER_CPPNODE( csp::python, _np_ema_timewise ); +REGISTER_CPPNODE( csp::python, _np_ema_debias_alpha ); +REGISTER_CPPNODE( csp::python, _np_ema_debias_halflife ); +static PyModuleDef _cspnpstatsimpl_module + = { PyModuleDef_HEAD_INIT, "_cspnpstatsimpl", "_cspnpstatsimpl c++ module", -1, NULL, NULL, NULL, NULL, NULL }; -static PyModuleDef _cspnpstatsimpl_module = { - PyModuleDef_HEAD_INIT, - "_cspnpstatsimpl", - "_cspnpstatsimpl c++ module", - -1, - NULL, NULL, NULL, NULL, NULL -}; - -PyMODINIT_FUNC PyInit__cspnpstatsimpl(void) +PyMODINIT_FUNC PyInit__cspnpstatsimpl( void ) { - PyObject* m; + PyObject * m; - m = PyModule_Create( &_cspnpstatsimpl_module); + m = PyModule_Create( &_cspnpstatsimpl_module ); if( m == NULL ) return NULL; diff --git a/cpp/csp/python/cspstatsimpl.cpp b/cpp/csp/python/cspstatsimpl.cpp index c01eb033..eac4382b 100644 --- a/cpp/csp/python/cspstatsimpl.cpp +++ b/cpp/csp/python/cspstatsimpl.cpp @@ -2,60 +2,55 @@ #include // Data processing nodes -REGISTER_CPPNODE( csp::cppnodes, _tick_window_updates ); -REGISTER_CPPNODE( csp::cppnodes, _time_window_updates ); -REGISTER_CPPNODE( csp::cppnodes, _cross_sectional_as_list ); -REGISTER_CPPNODE( csp::cppnodes, _min_hit_by_tick ); -REGISTER_CPPNODE( csp::cppnodes, _in_sequence_check ); -REGISTER_CPPNODE( csp::cppnodes, _sync_nan_f ); +REGISTER_CPPNODE( csp::cppnodes, _tick_window_updates ); +REGISTER_CPPNODE( csp::cppnodes, _time_window_updates ); +REGISTER_CPPNODE( csp::cppnodes, _cross_sectional_as_list ); +REGISTER_CPPNODE( csp::cppnodes, _min_hit_by_tick ); +REGISTER_CPPNODE( csp::cppnodes, _in_sequence_check ); +REGISTER_CPPNODE( csp::cppnodes, _sync_nan_f ); // Base statistics -REGISTER_CPPNODE( csp::cppnodes, _count ); -REGISTER_CPPNODE( csp::cppnodes, _sum ); -REGISTER_CPPNODE( csp::cppnodes, _kahan_sum ); -REGISTER_CPPNODE( csp::cppnodes, _mean ); -REGISTER_CPPNODE( csp::cppnodes, _var ); -REGISTER_CPPNODE( csp::cppnodes, _first ); -REGISTER_CPPNODE( csp::cppnodes, _unique ); -REGISTER_CPPNODE( csp::cppnodes, _prod ); -REGISTER_CPPNODE( csp::cppnodes, _weighted_mean ); -REGISTER_CPPNODE( csp::cppnodes, _weighted_var ); -REGISTER_CPPNODE( csp::cppnodes, _covar ); -REGISTER_CPPNODE( csp::cppnodes, _weighted_covar ); -REGISTER_CPPNODE( csp::cppnodes, _corr ); -REGISTER_CPPNODE( csp::cppnodes, _weighted_corr ); -REGISTER_CPPNODE( csp::cppnodes, _sem ); -REGISTER_CPPNODE( csp::cppnodes, _weighted_sem ); -REGISTER_CPPNODE( csp::cppnodes, _last ); -REGISTER_CPPNODE( csp::cppnodes, _quantile ); -REGISTER_CPPNODE( csp::cppnodes, _min_max ); -REGISTER_CPPNODE( csp::cppnodes, _rank ); -REGISTER_CPPNODE( csp::cppnodes, _arg_min_max ); -REGISTER_CPPNODE( csp::cppnodes, _skew ); -REGISTER_CPPNODE( csp::cppnodes, _weighted_skew ); -REGISTER_CPPNODE( csp::cppnodes, _kurt ); -REGISTER_CPPNODE( csp::cppnodes, _weighted_kurt ); +REGISTER_CPPNODE( csp::cppnodes, _count ); +REGISTER_CPPNODE( csp::cppnodes, _sum ); +REGISTER_CPPNODE( csp::cppnodes, _kahan_sum ); +REGISTER_CPPNODE( csp::cppnodes, _mean ); +REGISTER_CPPNODE( csp::cppnodes, _var ); +REGISTER_CPPNODE( csp::cppnodes, _first ); +REGISTER_CPPNODE( csp::cppnodes, _unique ); +REGISTER_CPPNODE( csp::cppnodes, _prod ); +REGISTER_CPPNODE( csp::cppnodes, _weighted_mean ); +REGISTER_CPPNODE( csp::cppnodes, _weighted_var ); +REGISTER_CPPNODE( csp::cppnodes, _covar ); +REGISTER_CPPNODE( csp::cppnodes, _weighted_covar ); +REGISTER_CPPNODE( csp::cppnodes, _corr ); +REGISTER_CPPNODE( csp::cppnodes, _weighted_corr ); +REGISTER_CPPNODE( csp::cppnodes, _sem ); +REGISTER_CPPNODE( csp::cppnodes, _weighted_sem ); +REGISTER_CPPNODE( csp::cppnodes, _last ); +REGISTER_CPPNODE( csp::cppnodes, _quantile ); +REGISTER_CPPNODE( csp::cppnodes, _min_max ); +REGISTER_CPPNODE( csp::cppnodes, _rank ); +REGISTER_CPPNODE( csp::cppnodes, _arg_min_max ); +REGISTER_CPPNODE( csp::cppnodes, _skew ); +REGISTER_CPPNODE( csp::cppnodes, _weighted_skew ); +REGISTER_CPPNODE( csp::cppnodes, _kurt ); +REGISTER_CPPNODE( csp::cppnodes, _weighted_kurt ); // EMA nodes -REGISTER_CPPNODE( csp::cppnodes, _ema_compute ); -REGISTER_CPPNODE( csp::cppnodes, _ema_adjusted ); -REGISTER_CPPNODE( csp::cppnodes, _ema_timewise ); -REGISTER_CPPNODE( csp::cppnodes, _ema_debias_alpha ); -REGISTER_CPPNODE( csp::cppnodes, _ema_debias_halflife ); +REGISTER_CPPNODE( csp::cppnodes, _ema_compute ); +REGISTER_CPPNODE( csp::cppnodes, _ema_adjusted ); +REGISTER_CPPNODE( csp::cppnodes, _ema_timewise ); +REGISTER_CPPNODE( csp::cppnodes, _ema_debias_alpha ); +REGISTER_CPPNODE( csp::cppnodes, _ema_debias_halflife ); -static PyModuleDef _cspstatsimpl_module = { - PyModuleDef_HEAD_INIT, - "_cspstatsimpl", - "_cspstatsimpl c++ module", - -1, - NULL, NULL, NULL, NULL, NULL -}; +static PyModuleDef _cspstatsimpl_module + = { PyModuleDef_HEAD_INIT, "_cspstatsimpl", "_cspstatsimpl c++ module", -1, NULL, NULL, NULL, NULL, NULL }; -PyMODINIT_FUNC PyInit__cspstatsimpl(void) +PyMODINIT_FUNC PyInit__cspstatsimpl( void ) { - PyObject* m; + PyObject * m; - m = PyModule_Create( &_cspstatsimpl_module); + m = PyModule_Create( &_cspstatsimpl_module ); if( m == NULL ) return NULL; diff --git a/cpp/csp/python/csptestlibimpl.cpp b/cpp/csp/python/csptestlibimpl.cpp index f5a750bf..ec09beb3 100644 --- a/cpp/csp/python/csptestlibimpl.cpp +++ b/cpp/csp/python/csptestlibimpl.cpp @@ -1,9 +1,9 @@ -#include #include #include #include #include #include +#include namespace csp::cppnodes { @@ -14,111 +14,91 @@ namespace csp::cppnodes namespace testing { -namespace stop_start_test -{ + namespace stop_start_test + { -using namespace csp::python; + using namespace csp::python; -void setStatus( const DialectGenericType & obj_, const std::string & name ) -{ - PyObjectPtr obj = PyObjectPtr::own( toPython( obj_ ) ); - PyObjectPtr attr = PyObjectPtr::own( PyUnicode_FromString( name.c_str() ) ); - PyObject_SetAttr( obj.get(), attr.get(), Py_True ); -} + void setStatus( const DialectGenericType & obj_, const std::string & name ) + { + PyObjectPtr obj = PyObjectPtr::own( toPython( obj_ ) ); + PyObjectPtr attr = PyObjectPtr::own( PyUnicode_FromString( name.c_str() ) ); + PyObject_SetAttr( obj.get(), attr.get(), Py_True ); + } -DECLARE_CPPNODE( start_n1_set_value ) -{ - INIT_CPPNODE( start_n1_set_value ) {} + DECLARE_CPPNODE( start_n1_set_value ) + { + INIT_CPPNODE( start_n1_set_value ) {} - SCALAR_INPUT( DialectGenericType, obj_ ); + SCALAR_INPUT( DialectGenericType, obj_ ); - START() - { - setStatus( obj_, "n1_started" ); - } - INVOKE() {} + START() { setStatus( obj_, "n1_started" ); } + INVOKE() {} - STOP() - { - setStatus( obj_, "n1_stopped" ); - } -}; -EXPORT_CPPNODE( start_n1_set_value ); + STOP() { setStatus( obj_, "n1_stopped" ); } + }; + EXPORT_CPPNODE( start_n1_set_value ); -DECLARE_CPPNODE( start_n2_throw ) -{ - INIT_CPPNODE( start_n2_throw ) {} + DECLARE_CPPNODE( start_n2_throw ) + { + INIT_CPPNODE( start_n2_throw ) {} - SCALAR_INPUT( DialectGenericType, obj_ ); + SCALAR_INPUT( DialectGenericType, obj_ ); - START() - { - CSP_THROW( ValueError, "n2 start failed" ); - } - INVOKE() {} + START() { CSP_THROW( ValueError, "n2 start failed" ); } + INVOKE() {} - STOP() - { - setStatus( obj_, "n2_stopped" ); - } -}; -EXPORT_CPPNODE( start_n2_throw ); + STOP() { setStatus( obj_, "n2_stopped" ); } + }; + EXPORT_CPPNODE( start_n2_throw ); -} + } // namespace stop_start_test -namespace interrupt_stop_test -{ + namespace interrupt_stop_test + { -using namespace csp::python; + using namespace csp::python; -void setStatus( const DialectGenericType & obj_, int64_t idx ) -{ - PyObjectPtr obj = PyObjectPtr::own( toPython( obj_ ) ); - PyObjectPtr list = PyObjectPtr::own( PyObject_GetAttrString( obj.get(), "stopped" ) ); - PyList_SET_ITEM( list.get(), idx, Py_True ); -} + void setStatus( const DialectGenericType & obj_, int64_t idx ) + { + PyObjectPtr obj = PyObjectPtr::own( toPython( obj_ ) ); + PyObjectPtr list = PyObjectPtr::own( PyObject_GetAttrString( obj.get(), "stopped" ) ); + PyList_SET_ITEM( list.get(), idx, Py_True ); + } -DECLARE_CPPNODE( set_stop_index ) -{ - INIT_CPPNODE( set_stop_index ) {} + DECLARE_CPPNODE( set_stop_index ) + { + INIT_CPPNODE( set_stop_index ) {} - SCALAR_INPUT( DialectGenericType, obj_ ); - SCALAR_INPUT( int64_t, idx ); + SCALAR_INPUT( DialectGenericType, obj_ ); + SCALAR_INPUT( int64_t, idx ); - START() {} - INVOKE() {} + START() {} + INVOKE() {} - STOP() - { - setStatus( obj_, idx ); - } -}; -EXPORT_CPPNODE( set_stop_index ); + STOP() { setStatus( obj_, idx ); } + }; + EXPORT_CPPNODE( set_stop_index ); -} + } // namespace interrupt_stop_test -} +} // namespace testing -} +} // namespace csp::cppnodes // Test nodes REGISTER_CPPNODE( csp::cppnodes::testing::stop_start_test, start_n1_set_value ); REGISTER_CPPNODE( csp::cppnodes::testing::stop_start_test, start_n2_throw ); REGISTER_CPPNODE( csp::cppnodes::testing::interrupt_stop_test, set_stop_index ); -static PyModuleDef _csptestlibimpl_module = { - PyModuleDef_HEAD_INIT, - "_csptestlibimpl", - "_csptestlibimpl c++ module", - -1, - NULL, NULL, NULL, NULL, NULL -}; +static PyModuleDef _csptestlibimpl_module + = { PyModuleDef_HEAD_INIT, "_csptestlibimpl", "_csptestlibimpl c++ module", -1, NULL, NULL, NULL, NULL, NULL }; -PyMODINIT_FUNC PyInit__csptestlibimpl(void) +PyMODINIT_FUNC PyInit__csptestlibimpl( void ) { - PyObject* m; + PyObject * m; - m = PyModule_Create( &_csptestlibimpl_module); + m = PyModule_Create( &_csptestlibimpl_module ); if( m == NULL ) return NULL; diff --git a/cpp/csp/python/csptypesimpl.cpp b/cpp/csp/python/csptypesimpl.cpp index a0a5ec45..4cc177b0 100644 --- a/cpp/csp/python/csptypesimpl.cpp +++ b/cpp/csp/python/csptypesimpl.cpp @@ -5,19 +5,14 @@ namespace csp::python { -static PyModuleDef _csptypesimpl_module = { - PyModuleDef_HEAD_INIT, - "_csptypesimpl", - "_csptypesimpl c++ module", - -1, - NULL, NULL, NULL, NULL, NULL -}; +static PyModuleDef _csptypesimpl_module + = { PyModuleDef_HEAD_INIT, "_csptypesimpl", "_csptypesimpl c++ module", -1, NULL, NULL, NULL, NULL, NULL }; -PyMODINIT_FUNC PyInit__csptypesimpl(void) +PyMODINIT_FUNC PyInit__csptypesimpl( void ) { - PyObject* m; + PyObject * m; - m = PyModule_Create( &_csptypesimpl_module); + m = PyModule_Create( &_csptypesimpl_module ); if( m == NULL ) return NULL; @@ -27,4 +22,4 @@ PyMODINIT_FUNC PyInit__csptypesimpl(void) return m; } -} +} // namespace csp::python diff --git a/cpp/csp/python/npstatsimpl.cpp b/cpp/csp/python/npstatsimpl.cpp index 518afd79..13c683fe 100644 --- a/cpp/csp/python/npstatsimpl.cpp +++ b/cpp/csp/python/npstatsimpl.cpp @@ -1,28 +1,39 @@ #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#include // needs to be included first #include #include // need to be included before csp +#include // needs to be included first -//Include first so that csp dialect types are defined +// Include first so that csp dialect types are defined #include #include -#include #include +#include #include #include #include -#include #include #define NPY_SHAPE_ERROR "Shape of the NumPy array was unknown at the time the trigger ticked." // Need to copy these from NumpyConversions.h due to the autogen build // autogen types rely on _cspimpl.so files to be created first -template struct NPY_TYPE { static const int value = NPY_OBJECT; }; -template<> struct NPY_TYPE { static const int value = NPY_LONGLONG; }; -template<> struct NPY_TYPE { static const int value = NPY_DOUBLE; }; +template +struct NPY_TYPE +{ + static const int value = NPY_OBJECT; +}; +template<> +struct NPY_TYPE +{ + static const int value = NPY_LONGLONG; +}; +template<> +struct NPY_TYPE +{ + static const int value = NPY_DOUBLE; +}; using namespace csp::cppnodes; @@ -45,108 +56,105 @@ template class NumPyIterator { // Code to iterate over an arbitrary NumPy array - // Iterates over the array in a non-necessarily contiguous order, so all operations must be defined only element-wise + // Iterates over the array in a non-necessarily contiguous order, so all operations must be defined only + // element-wise - public: - - NumPyIterator() - { - m_nd = 0; - m_size = 0; - m_index = 0; - m_data = nullptr; - m_strides = nullptr; - m_stridedDimensions = {}; - m_current = {}; - m_valid = false; - } +public: + NumPyIterator() + { + m_nd = 0; + m_size = 0; + m_index = 0; + m_data = nullptr; + m_strides = nullptr; + m_stridedDimensions = {}; + m_current = {}; + m_valid = false; + } - NumPyIterator( PyArrayObject * arr ) - { - setup( arr ); - } + NumPyIterator( PyArrayObject * arr ) { setup( arr ); } - NumPyIterator( PyObject * arr ) - { - if( unlikely( !PyArray_Check( arr ) ) ) - CSP_THROW( csp::TypeError, "Expected NumPy array type, got " << Py_TYPE( arr ) -> tp_name ); - setup( ( PyArrayObject * ) arr ); - } + NumPyIterator( PyObject * arr ) + { + if( unlikely( !PyArray_Check( arr ) ) ) + CSP_THROW( csp::TypeError, "Expected NumPy array type, got " << Py_TYPE( arr )->tp_name ); + setup( (PyArrayObject *)arr ); + } - NumPyIterator( NumPyIterator && rhs ) = default; + NumPyIterator( NumPyIterator && rhs ) = default; - NumPyIterator & operator=( NumPyIterator && rhs ) = default; + NumPyIterator & operator=( NumPyIterator && rhs ) = default; - NumPyIterator( const NumPyIterator & rhs ) = delete; + NumPyIterator( const NumPyIterator & rhs ) = delete; - NumPyIterator & operator=( const NumPyIterator & rhs ) = delete; + NumPyIterator & operator=( const NumPyIterator & rhs ) = delete; - operator bool() const { return m_valid; } + operator bool() const { return m_valid; } - void operator++() + void operator++() + { + for( int i = m_nd - 1; i >= 0; --i ) { - for( int i = m_nd - 1; i >=0 ; --i ) + if( ++m_current[i] < m_dims[i] ) { - if( ++m_current[i] < m_dims[i] ) - { - m_data += m_strides[i];; - m_index++; - return; - } - m_data -= m_stridedDimensions[i]; - m_current[i] = 0; + m_data += m_strides[i]; + ; + m_index++; + return; } - m_valid = false; + m_data -= m_stridedDimensions[i]; + m_current[i] = 0; } + m_valid = false; + } - int64_t index() const { return m_index; } - - const npy_intp * indices() const { return (npy_intp*)m_current.data(); } + int64_t index() const { return m_index; } - T & value() const { return *reinterpret_cast( m_data ); } + const npy_intp * indices() const { return (npy_intp *)m_current.data(); } - private: + T & value() const { return *reinterpret_cast( m_data ); } - void verify_arr( PyArrayObject * arr ) +private: + void verify_arr( PyArrayObject * arr ) + { + auto expType = PyArray_DescrFromType( NPY_TYPE::value ); + if( unlikely( PyObject_RichCompareBool( (PyObject *)PyArray_DESCR( arr ), (PyObject *)expType, Py_EQ ) ) != 1 ) { - auto expType = PyArray_DescrFromType( NPY_TYPE::value ); - if( unlikely( PyObject_RichCompareBool( ( PyObject * ) PyArray_DESCR( arr ), ( PyObject * ) expType, Py_EQ ) ) != 1 ) - { - CSP_THROW( csp::TypeError, - "Expected array of type " << PyObjectPtr::own( PyObject_Repr( ( PyObject * ) expType ) ) - << " got " - << PyObjectPtr::own( PyObject_Repr( ( PyObject * ) PyArray_DESCR( arr ) ) ) ); - } + CSP_THROW( csp::TypeError, + "Expected array of type " + << PyObjectPtr::own( PyObject_Repr( (PyObject *)expType ) ) << " got " + << PyObjectPtr::own( PyObject_Repr( (PyObject *)PyArray_DESCR( arr ) ) ) ); } + } - void setup( PyArrayObject * arr ) - { - verify_arr( arr ); - m_index = 0; - m_nd = PyArray_NDIM( arr ); - m_current.resize( m_nd, 0 ); - // set strides and dimension - m_dims = PyArray_DIMS( arr ); - m_strides = PyArray_STRIDES( arr ); - for( int i = 0; i < m_nd; ++i ) - m_stridedDimensions.emplace_back( m_strides[i] * ( m_dims[i] - 1 ) ); - m_data = reinterpret_cast( PyArray_DATA( arr ) ); - if( m_nd == 0 ) - m_size = 0; - else - m_size = std::accumulate( m_dims, m_dims + m_nd, 1, std::multiplies() ); - m_valid = ( m_size > 0 ); - } + void setup( PyArrayObject * arr ) + { + verify_arr( arr ); + m_index = 0; + m_nd = PyArray_NDIM( arr ); + m_current.resize( m_nd, 0 ); + // set strides and dimension + m_dims = PyArray_DIMS( arr ); + m_strides = PyArray_STRIDES( arr ); + for( int i = 0; i < m_nd; ++i ) + m_stridedDimensions.emplace_back( m_strides[i] * ( m_dims[i] - 1 ) ); + m_data = reinterpret_cast( PyArray_DATA( arr ) ); + if( m_nd == 0 ) + m_size = 0; + else + m_size = std::accumulate( m_dims, m_dims + m_nd, 1, std::multiplies() ); + m_valid = ( m_size > 0 ); + } - int64_t m_nd; - int64_t m_size; - int64_t m_index; - char* m_data; - npy_intp* m_strides; - npy_intp * m_dims; - std::vector m_stridedDimensions; - std::vector m_current; - bool m_valid; + int64_t m_nd; + int64_t m_size; + int64_t m_index; + char * m_data; + npy_intp * m_strides; + npy_intp * m_dims; + std::vector m_stridedDimensions; + std::vector m_current; + bool m_valid; }; struct PyShape @@ -154,14 +162,14 @@ struct PyShape PyShape() { m_dims = {}; - m_n = 0; + m_n = 0; } - PyShape( PyArrayObject* arr ) + PyShape( PyArrayObject * arr ) { int64_t nd = PyArray_NDIM( arr ); m_dims.reserve( nd ); - npy_intp* dims = PyArray_DIMS( arr ); + npy_intp * dims = PyArray_DIMS( arr ); for( int64_t j = 0; j < nd; j++ ) m_dims.emplace_back( dims[j] ); if( nd == 0 ) @@ -170,37 +178,41 @@ struct PyShape m_n = std::accumulate( std::begin( m_dims ), std::end( m_dims ), 1, std::multiplies() ); } - PyShape( PyObject* arr ) : PyShape( ( PyArrayObject* ) arr ) { } + PyShape( PyObject * arr ) + : PyShape( (PyArrayObject *)arr ) + { + } PyShape( std::vector dims, int64_t n ) { m_dims = dims; - m_n = n; + m_n = n; } - void validateShape( PyArrayObject* arr ) + void validateShape( PyArrayObject * arr ) { size_t nd = PyArray_NDIM( arr ); if( nd != m_dims.size() ) - CSP_THROW( ValueError, "Inconsistent shape of NumPy arrays for computation: different number of dimensions"); + CSP_THROW( ValueError, + "Inconsistent shape of NumPy arrays for computation: different number of dimensions" ); - npy_intp* dims = PyArray_DIMS( arr ); + npy_intp * dims = PyArray_DIMS( arr ); for( size_t i = 0; i < nd; i++ ) { if( dims[i] != m_dims[i] ) - CSP_THROW( ValueError, "Inconsistent shape of NumPy arrays for computation: difference in dimension " << i ); + CSP_THROW( ValueError, + "Inconsistent shape of NumPy arrays for computation: difference in dimension " << i ); } } - void validateShape( PyObject* arr ) - { - validateShape( ( PyArrayObject* ) arr ); - } + void validateShape( PyObject * arr ) { validateShape( (PyArrayObject *)arr ); } bool operator!=( const PyShape & rhs ) const { - if( m_n != rhs.m_n ) return true; - if( m_dims.size() != rhs.m_dims.size() ) return true; + if( m_n != rhs.m_n ) + return true; + if( m_dims.size() != rhs.m_dims.size() ) + return true; for( size_t i = 0; i < m_dims.size(); i++ ) { if( m_dims[i] != rhs.m_dims[i] ) @@ -213,32 +225,32 @@ struct PyShape std::vector m_dims; int64_t m_n; - }; -PyObject* createNanWithShape( PyObject* t ) +PyObject * createNanWithShape( PyObject * t ) { - PyArrayObject* as_pyarr = ( PyArrayObject* )t; - int64_t nd = PyArray_NDIM( as_pyarr ); - PyObject* ret = PyArray_EMPTY( nd, PyArray_DIMS( as_pyarr ), NPY_DOUBLE, 0 ); - PyArray_FillWithScalar( ( PyArrayObject* ) ret, ( PyObject* ) PyFloat_FromDouble( std::numeric_limits::quiet_NaN() ) ); + PyArrayObject * as_pyarr = (PyArrayObject *)t; + int64_t nd = PyArray_NDIM( as_pyarr ); + PyObject * ret = PyArray_EMPTY( nd, PyArray_DIMS( as_pyarr ), NPY_DOUBLE, 0 ); + PyArray_FillWithScalar( (PyArrayObject *)ret, + (PyObject *)PyFloat_FromDouble( std::numeric_limits::quiet_NaN() ) ); return ret; } -PyObject* createZerosWithShape( PyObject* t ) +PyObject * createZerosWithShape( PyObject * t ) { - PyArrayObject* as_pyarr = ( PyArrayObject* )t; - int64_t nd = PyArray_NDIM( as_pyarr ); + PyArrayObject * as_pyarr = (PyArrayObject *)t; + int64_t nd = PyArray_NDIM( as_pyarr ); return PyArray_Zeros( nd, PyArray_DIMS( as_pyarr ), PyArray_DescrFromType( NPY_DOUBLE ), 0 ); } // C: computation class template -inline PyObject* computeArray( const PyShape & shp, std::vector & elem, bool s_first ) +inline PyObject * computeArray( const PyShape & shp, std::vector & elem, bool s_first ) { if( unlikely( s_first ) ) CSP_THROW( ValueError, NPY_SHAPE_ERROR ); - PyObject* out = PyArray_EMPTY( shp.m_dims.size(), &shp.m_dims[0], NPY_DOUBLE, 0 ); + PyObject * out = PyArray_EMPTY( shp.m_dims.size(), &shp.m_dims[0], NPY_DOUBLE, 0 ); for( NumPyIterator iter( out ); iter; ++iter ) iter.value() = elem[iter.index()].compute(); @@ -255,14 +267,11 @@ class _np_tick_window_updates : public _generic_tick_window_updates x ) ) - CSP_THROW( ValueError, "Error: sampler called on a NumPy array before any data ticks - shape is unknown."); + if( !csp.valid( this->x ) ) + CSP_THROW( ValueError, "Error: sampler called on a NumPy array before any data ticks - shape is unknown." ); } }; @@ -282,19 +291,16 @@ EXPORT_CPPNODE( _np_tick_window_updates ); class _np_time_window_updates : public _generic_time_window_updates { public: - using _generic_time_window_updates::_generic_time_window_updates; + using _generic_time_window_updates::_generic_time_window_updates; _STATIC_CREATE_METHOD( _np_time_window_updates ); STATE_VAR( PyShape, s_shp{} ); - inline PyObjectPtr createNan() - { - return PyObjectPtr::own( createNanWithShape( x.lastValue().get() ) ); - } + inline PyObjectPtr createNan() { return PyObjectPtr::own( createNanWithShape( x.lastValue().get() ) ); } inline void validateShape() { - PyObject* arr = x.lastValue().get(); + PyObject * arr = x.lastValue().get(); PyShape shp( arr ); if( unlikely( s_first ) ) s_shp = shp; @@ -304,8 +310,8 @@ class _np_time_window_updates : public _generic_time_window_updates x ) ) - CSP_THROW( ValueError, "Error: sampler called on a NumPy array before any data ticks - shape is unknown."); + if( !csp.valid( this->x ) ) + CSP_THROW( ValueError, "Error: sampler called on a NumPy array before any data ticks - shape is unknown." ); } }; @@ -319,9 +325,9 @@ class _cross_sectional_as_np : public _generic_cross_sectional( PyArray_DATA( ( PyArrayObject* )out ) ); + npy_intp n = s_window.count(); + PyObject * out = PyArray_EMPTY( 1, &n, NPY_DOUBLE, 0 ); // 1D array + double * data = reinterpret_cast( PyArray_DATA( (PyArrayObject *)out ) ); s_window.copy_values( data ); RETURN( PyObjectPtr::own( out ) ); } @@ -329,16 +335,15 @@ class _cross_sectional_as_np : public _generic_cross_sectional, _np_cross_sectional_as_list> +class _np_cross_sectional_as_list + : public _generic_cross_sectional, _np_cross_sectional_as_list> { public: - using _generic_cross_sectional, _np_cross_sectional_as_list>::_generic_cross_sectional; + using _generic_cross_sectional, + _np_cross_sectional_as_list>::_generic_cross_sectional; _STATIC_CREATE_METHOD( _np_cross_sectional_as_list ); - inline void computeCrossSectional() - { - s_window.copy_values( &unnamed_output().reserveSpace() ); - } + inline void computeCrossSectional() { s_window.copy_values( &unnamed_output().reserveSpace() ); } }; EXPORT_CPPNODE( _np_cross_sectional_as_list ); @@ -352,26 +357,26 @@ class _np_cross_sectional_as_np : public _generic_cross_sectional::const_iterator it = s_window.left(); - PyArrayObject* fval = ( PyArrayObject* )it.value().get(); - int64_t nd = PyArray_NDIM( fval ); - npy_intp* inner_dims = PyArray_DIMS( fval ); - npy_intp* dims = new npy_intp[nd+1]; - dims[0] = s_window.count(); + PyArrayObject * fval = (PyArrayObject *)it.value().get(); + int64_t nd = PyArray_NDIM( fval ); + npy_intp * inner_dims = PyArray_DIMS( fval ); + npy_intp * dims = new npy_intp[nd + 1]; + dims[0] = s_window.count(); for( int64_t i = 0; i < nd; i++ ) - dims[i+1] = inner_dims[i]; + dims[i + 1] = inner_dims[i]; - out = PyArray_EMPTY( nd+1, dims, NPY_DOUBLE, 0 ); - double* data = reinterpret_cast( PyArray_DATA( ( PyArrayObject* )out ) ); + out = PyArray_EMPTY( nd + 1, dims, NPY_DOUBLE, 0 ); + double * data = reinterpret_cast( PyArray_DATA( (PyArrayObject *)out ) ); int64_t np_index = 0; // Copy directly from buffer for( int64_t i = 0; i < s_window.count(); i++ ) { - PyObject* v = it.value().get(); // reference + PyObject * v = it.value().get(); // reference for( NumPyIterator iter( v ); iter; ++iter ) { data[np_index] = iter.value(); @@ -383,7 +388,7 @@ class _np_cross_sectional_as_np : public _generic_cross_sectional( PyArray_DATA( ( PyArrayObject* )out ) ); + npy_intp n = x.size(); + PyObject * out = PyArray_EMPTY( 1, &n, NPY_DOUBLE, 0 ); + double * data = reinterpret_cast( PyArray_DATA( (PyArrayObject *)out ) ); for( int64_t i = 0; i < n; i++ ) { if( csp.valid( x[i] ) && ( fillna || csp.ticked( x[i] ) ) ) @@ -426,28 +431,29 @@ DECLARE_CPPNODE( _np_to_list ) TS_LISTBASKET_OUTPUT( double ); - INIT_CPPNODE( _np_to_list ) { } + INIT_CPPNODE( _np_to_list ) {} START() { - if( n == 0) - CSP_THROW( ValueError, "Must provide at least one output channel for NumPy conversion"); + if( n == 0 ) + CSP_THROW( ValueError, "Must provide at least one output channel for NumPy conversion" ); } INVOKE() { - PyArrayObject* arr = ( PyArrayObject* )x.lastValue().get(); + PyArrayObject * arr = (PyArrayObject *)x.lastValue().get(); // Ensure shape is valid int64_t nd = PyArray_NDIM( arr ); if( nd != 1 ) - CSP_THROW( ValueError, "Cannot convert NumPy array of more than one dimension into listbasket"); + CSP_THROW( ValueError, "Cannot convert NumPy array of more than one dimension into listbasket" ); int64_t n_arr = PyArray_SIZE( arr ); if( n_arr != n ) - CSP_THROW( ValueError, "Cannot convert NumPy array of size " << n_arr << " into listbasket of " << n << " elements"); + CSP_THROW( ValueError, + "Cannot convert NumPy array of size " << n_arr << " into listbasket of " << n << " elements" ); - double* data = reinterpret_cast( PyArray_DATA( arr ) ); + double * data = reinterpret_cast( PyArray_DATA( arr ) ); for( int64_t i = 0; i < n; i++ ) unnamed_output()[i].output( data[i] ); } @@ -467,17 +473,17 @@ DECLARE_CPPNODE( _sync_nan_np ) STATE_VAR( PyShape, s_shp{} ); STATE_VAR( bool, s_first{ true } ); - INIT_CPPNODE( _sync_nan_np ) { } + INIT_CPPNODE( _sync_nan_np ) {} INVOKE() { // Note x and y are guaranteed to tick in sequence in contexts where sync_nan is used - PyArrayObject * xval = ( PyArrayObject* )x.lastValue().get(); - PyArrayObject * yval = ( PyArrayObject* )y.lastValue().get(); + PyArrayObject * xval = (PyArrayObject *)x.lastValue().get(); + PyArrayObject * yval = (PyArrayObject *)y.lastValue().get(); if( unlikely( s_first ) ) { - s_shp = PyShape( xval ); + s_shp = PyShape( xval ); s_first = false; } else @@ -488,27 +494,27 @@ DECLARE_CPPNODE( _sync_nan_np ) // Only allocate a new array if we need to change values PyArrayObject * xsync = nullptr; PyArrayObject * ysync = nullptr; - for( NumPyIterator itx = {xval}, ity = {yval}; itx && ity; ++itx, ++ity ) + for( NumPyIterator itx = { xval }, ity = { yval }; itx && ity; ++itx, ++ity ) { if( isnan( ity.value() ) && !isnan( itx.value() ) ) // need to set corresponding element in x as nan { if( !xsync ) { - xsync = ( PyArrayObject* )PyArray_NewLikeArray( xval, NPY_KEEPORDER, NULL, 0 ); + xsync = (PyArrayObject *)PyArray_NewLikeArray( xval, NPY_KEEPORDER, NULL, 0 ); PyArray_CopyInto( xsync, xval ); } - double * element = ( double * )PyArray_GetPtr( xsync, itx.indices() ); - *( reinterpret_cast( element ) ) = std::numeric_limits::quiet_NaN(); + double * element = (double *)PyArray_GetPtr( xsync, itx.indices() ); + *( reinterpret_cast( element ) ) = std::numeric_limits::quiet_NaN(); } else if( isnan( itx.value() ) && !isnan( ity.value() ) ) // need to set corresponding element in y as nan { if( !ysync ) { - ysync = ( PyArrayObject* )PyArray_NewLikeArray( yval, NPY_KEEPORDER, NULL, 0 ); + ysync = (PyArrayObject *)PyArray_NewLikeArray( yval, NPY_KEEPORDER, NULL, 0 ); PyArray_CopyInto( ysync, yval ); } - double * element = ( double * )PyArray_GetPtr( ysync, ity.indices() ); - *element = std::numeric_limits::quiet_NaN(); + double * element = (double *)PyArray_GetPtr( ysync, ity.indices() ); + *element = std::numeric_limits::quiet_NaN(); } } @@ -523,8 +529,8 @@ DECLARE_CPPNODE( _sync_nan_np ) Py_XINCREF( yval ); } - x_sync.output( PyObjectPtr::own( ( PyObject * )xsync ) ); - y_sync.output( PyObjectPtr::own( ( PyObject * )ysync ) ); + x_sync.output( PyObjectPtr::own( (PyObject *)xsync ) ); + y_sync.output( PyObjectPtr::own( (PyObject *)ysync ) ); } }; @@ -544,16 +550,18 @@ DECLARE_CPPNODE( _np_compute ) STATE_VAR( std::vector>, s_elem ); STATE_VAR( PyShape, s_shp{} ); - STATE_VAR( bool, s_first{true} ); + STATE_VAR( bool, s_first{ true } ); TS_OUTPUT( PyObjectPtr ); - //Expanded out INIT_CPPNODE without create call... + // Expanded out INIT_CPPNODE without create call... CSP csp; const char * name() const override { return "_np_compute"; } public: - _np_compute( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} + _np_compute( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) + { + } virtual DataValidator initDataValidator() = 0; @@ -568,8 +576,8 @@ DECLARE_CPPNODE( _np_compute ) { if( unlikely( s_first ) ) { - PyObject* arr = additions.lastValue()[0].get(); - s_shp = PyShape( arr ); + PyObject * arr = additions.lastValue()[0].get(); + s_shp = PyShape( arr ); s_elem.reserve( s_shp.m_n ); for( int j = 0; j < s_shp.m_n; j++ ) { @@ -581,7 +589,7 @@ DECLARE_CPPNODE( _np_compute ) size_t m = additions.lastValue().size(); for( size_t i = 0; i < m; i++ ) { - PyObject* v = additions.lastValue()[i].get(); + PyObject * v = additions.lastValue()[i].get(); for( NumPyIterator iter( v ); iter; ++iter ) { s_elem[iter.index()].add( iter.value() ); @@ -593,7 +601,7 @@ DECLARE_CPPNODE( _np_compute ) size_t m = removals.lastValue().size(); for( size_t i = 0; i < m; i++ ) { - PyObject* v = removals.lastValue()[i].get(); + PyObject * v = removals.lastValue()[i].get(); for( NumPyIterator iter( v ); iter; ++iter ) s_elem[iter.index()].remove( iter.value() ); } @@ -612,10 +620,7 @@ class _npComputeCommonArgs : public _np_compute using _np_compute::_np_compute; _STATIC_CREATE_METHOD( _npComputeCommonArgs ); - DataValidator initDataValidator() override - { - return DataValidator( this -> min_data_points, this -> ignore_na ); - } + DataValidator initDataValidator() override { return DataValidator( this->min_data_points, this->ignore_na ); } }; template @@ -628,7 +633,7 @@ class _npComputeOneArg : public _np_compute DataValidator initDataValidator() override { - return DataValidator( this -> min_data_points, this -> ignore_na, this -> arg ); + return DataValidator( this->min_data_points, this->ignore_na, this->arg ); } }; @@ -643,7 +648,7 @@ class _npComputeTwoArg : public _np_compute DataValidator initDataValidator() override { - return DataValidator( this -> min_data_points, this -> ignore_na, this -> arg1, this -> arg2 ); + return DataValidator( this->min_data_points, this->ignore_na, this->arg1, this->arg2 ); } }; @@ -653,35 +658,34 @@ class _npComputeEMA : public _np_compute public: using _np_compute::_np_compute; _STATIC_CREATE_METHOD( _npComputeEMA ); - SCALAR_INPUT( double, alpha ); + SCALAR_INPUT( double, alpha ); SCALAR_INPUT( int64_t, horizon ); SCALAR_INPUT( bool, adjust ); DataValidator initDataValidator() override { - return DataValidator( this -> min_data_points, true, alpha, this -> ignore_na, horizon, adjust ); + return DataValidator( this->min_data_points, true, alpha, this->ignore_na, horizon, adjust ); } }; - // Export node templates -EXPORT_TEMPLATE_CPPNODE( _np_count, _npComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _np_sum, _npComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _np_kahan_sum, _npComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _np_mean, _npComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _np_prod, _npComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _np_first, _npComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _np_last, _npComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _np_unique, SINGLE_ARG( _npComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_var, SINGLE_ARG( _npComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_sem, SINGLE_ARG( _npComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_min_max, SINGLE_ARG( _npComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_skew, SINGLE_ARG( _npComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_rank, SINGLE_ARG( _npComputeTwoArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_kurt, SINGLE_ARG( _npComputeTwoArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_ema_compute, _npComputeEMA ); -EXPORT_TEMPLATE_CPPNODE( _np_ema_adjusted, _npComputeEMA); -EXPORT_TEMPLATE_CPPNODE( _np_ema_debias_alpha, _npComputeEMA ); +EXPORT_TEMPLATE_CPPNODE( _np_count, _npComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _np_sum, _npComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _np_kahan_sum, _npComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _np_mean, _npComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _np_prod, _npComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _np_first, _npComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _np_last, _npComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _np_unique, SINGLE_ARG( _npComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_var, SINGLE_ARG( _npComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_sem, SINGLE_ARG( _npComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_min_max, SINGLE_ARG( _npComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_skew, SINGLE_ARG( _npComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_rank, SINGLE_ARG( _npComputeTwoArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_kurt, SINGLE_ARG( _npComputeTwoArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_ema_compute, _npComputeEMA ); +EXPORT_TEMPLATE_CPPNODE( _np_ema_adjusted, _npComputeEMA ); +EXPORT_TEMPLATE_CPPNODE( _np_ema_debias_alpha, _npComputeEMA ); // Bivariate template @@ -699,17 +703,19 @@ DECLARE_CPPNODE( _np_bivariate ) STATE_VAR( std::vector>, s_elem{} ); STATE_VAR( PyShape, s_shp{} ); - STATE_VAR( bool, s_first{true} ); + STATE_VAR( bool, s_first{ true } ); TS_OUTPUT( PyObjectPtr ); - //Expanded out INIT_CPPNODE without create call... + // Expanded out INIT_CPPNODE without create call... CSP csp; const char * name() const override { return "_np_bivariate"; } public: - _np_bivariate( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} + _np_bivariate( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) + { + } virtual DataValidator initDataValidator() = 0; @@ -722,13 +728,13 @@ DECLARE_CPPNODE( _np_bivariate ) } if( csp.ticked( x_add ) ) { - const std::vector & add_x = x_add.lastValue(); + const std::vector & add_x = x_add.lastValue(); const std::vector & weights = w_add.lastValue(); if( unlikely( s_first ) ) { - PyObject* arr = add_x[0].get(); - s_shp = PyShape( arr ); + PyObject * arr = add_x[0].get(); + s_shp = PyShape( arr ); s_elem.reserve( s_shp.m_n ); for( int64_t j = 0; j < s_shp.m_n; j++ ) { @@ -740,11 +746,11 @@ DECLARE_CPPNODE( _np_bivariate ) size_t m = add_x.size(); for( size_t i = 0; i < m; i++ ) { - PyObject* v = add_x[i].get(); - PyObject* w = weights[i].get(); + PyObject * v = add_x[i].get(); + PyObject * w = weights[i].get(); s_shp.validateShape( w ); - for( NumPyIterator iter = {v}, w_iter = {w}; iter && w_iter; ++iter, ++w_iter ) + for( NumPyIterator iter = { v }, w_iter = { w }; iter && w_iter; ++iter, ++w_iter ) { s_elem[iter.index()].add( iter.value(), w_iter.value() ); } @@ -753,16 +759,16 @@ DECLARE_CPPNODE( _np_bivariate ) if( csp.ticked( x_rem ) ) { - const std::vector & rem_x = x_rem.lastValue(); + const std::vector & rem_x = x_rem.lastValue(); const std::vector & weights = w_rem.lastValue(); size_t m = rem_x.size(); for( size_t i = 0; i < m; i++ ) { - PyObject* v = rem_x[i].get(); - PyObject* w = weights[i].get(); + PyObject * v = rem_x[i].get(); + PyObject * w = weights[i].get(); - for( NumPyIterator iter = {v}, w_iter = {w}; iter && w_iter; ++iter, ++w_iter ) + for( NumPyIterator iter = { v }, w_iter = { w }; iter && w_iter; ++iter, ++w_iter ) s_elem[iter.index()].remove( iter.value(), w_iter.value() ); } } @@ -781,10 +787,7 @@ class _npBivarCompute : public _np_bivariate using _np_bivariate::_np_bivariate; _STATIC_CREATE_METHOD( _npBivarCompute ); - DataValidator initDataValidator() override - { - return DataValidator( this -> min_data_points, this -> ignore_na ); - } + DataValidator initDataValidator() override { return DataValidator( this->min_data_points, this->ignore_na ); } }; template @@ -797,11 +800,10 @@ class _npBivarComputeOneArg : public _np_bivariate DataValidator initDataValidator() override { - return DataValidator( this -> min_data_points, this -> ignore_na, this -> arg ); + return DataValidator( this->min_data_points, this->ignore_na, this->arg ); } }; - template class _npBivarComputeTwoArg : public _np_bivariate { @@ -813,27 +815,28 @@ class _npBivarComputeTwoArg : public _np_bivariate DataValidator initDataValidator() override { - return DataValidator( this -> min_data_points, this -> ignore_na, this -> arg1, this -> arg2 ); + return DataValidator( this->min_data_points, this->ignore_na, this->arg1, this->arg2 ); } }; -EXPORT_TEMPLATE_CPPNODE( _np_weighted_mean, _npBivarCompute ); -EXPORT_TEMPLATE_CPPNODE( _np_corr, _npBivarCompute ); -EXPORT_TEMPLATE_CPPNODE( _np_weighted_var, SINGLE_ARG( _npBivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_covar, SINGLE_ARG( _npBivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_weighted_sem, SINGLE_ARG( _npBivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_weighted_skew, SINGLE_ARG( _npBivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _np_weighted_kurt, SINGLE_ARG( _npBivarComputeTwoArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_weighted_mean, _npBivarCompute ); +EXPORT_TEMPLATE_CPPNODE( _np_corr, _npBivarCompute ); +EXPORT_TEMPLATE_CPPNODE( _np_weighted_var, SINGLE_ARG( _npBivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_covar, SINGLE_ARG( _npBivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_weighted_sem, SINGLE_ARG( _npBivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_weighted_skew, SINGLE_ARG( _npBivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _np_weighted_kurt, SINGLE_ARG( _npBivarComputeTwoArg ) ); // Other node templates /* @csp.node -def _np_quantile(additions: ts[[np.ndarray]], removals: ts[[np.ndarray]], quants: typing.List[float], nq: int, interpolation_type: int, trigger: ts[object]): +def _np_quantile(additions: ts[[np.ndarray]], removals: ts[[np.ndarray]], quants: typing.List[float], nq: int, +interpolation_type: int, trigger: ts[object]): __outputs__(ts[np.ndarray]) */ -DECLARE_CPPNODE ( _np_quantile ) +DECLARE_CPPNODE( _np_quantile ) { TS_INPUT( std::vector, additions ); TS_INPUT( std::vector, removals ); @@ -846,11 +849,11 @@ DECLARE_CPPNODE ( _np_quantile ) STATE_VAR( std::vector>, s_elem ); STATE_VAR( PyShape, s_shp{} ); - STATE_VAR( bool, s_first{true} ); + STATE_VAR( bool, s_first{ true } ); TS_LISTBASKET_OUTPUT( PyObjectPtr ); - INIT_CPPNODE( _np_quantile ) { } + INIT_CPPNODE( _np_quantile ) {} INVOKE() { @@ -863,12 +866,13 @@ DECLARE_CPPNODE ( _np_quantile ) { if( unlikely( s_first ) ) { - PyObject* arr = additions.lastValue()[0].get(); - s_shp = PyShape( arr ); + PyObject * arr = additions.lastValue()[0].get(); + s_shp = PyShape( arr ); s_elem.reserve( s_shp.m_n ); for( int64_t j = 0; j < s_shp.m_n; j++ ) { - s_elem.emplace_back( DataValidator( min_data_points, ignore_na, quants, interpolation_type ) ); + s_elem.emplace_back( + DataValidator( min_data_points, ignore_na, quants, interpolation_type ) ); } s_first = false; } @@ -876,7 +880,7 @@ DECLARE_CPPNODE ( _np_quantile ) size_t m = additions.lastValue().size(); for( size_t i = 0; i < m; i++ ) { - PyObject* v = additions.lastValue()[i].get(); + PyObject * v = additions.lastValue()[i].get(); for( NumPyIterator iter( v ); iter; ++iter ) s_elem[iter.index()].add( iter.value() ); } @@ -887,7 +891,7 @@ DECLARE_CPPNODE ( _np_quantile ) size_t m = removals.lastValue().size(); for( size_t i = 0; i < m; i++ ) { - PyObject* v = removals.lastValue()[i].get(); + PyObject * v = removals.lastValue()[i].get(); for( NumPyIterator iter( v ); iter; ++iter ) s_elem[iter.index()].remove( iter.value() ); } @@ -898,7 +902,7 @@ DECLARE_CPPNODE ( _np_quantile ) int64_t nq = quants.value().size(); for( int64_t j = 0; j < nq; j++ ) { - PyObject* out = PyArray_EMPTY( s_shp.m_dims.size(), &s_shp.m_dims[0], NPY_DOUBLE, 0 ); + PyObject * out = PyArray_EMPTY( s_shp.m_dims.size(), &s_shp.m_dims[0], NPY_DOUBLE, 0 ); for( NumPyIterator iter( out ); iter; ++iter ) { iter.value() = s_elem[iter.index()].compute( j ); // compute jth quantile @@ -907,10 +911,9 @@ DECLARE_CPPNODE ( _np_quantile ) } } } - }; -EXPORT_CPPNODE ( _np_quantile ); +EXPORT_CPPNODE( _np_quantile ); // C: computation class template @@ -925,11 +928,11 @@ DECLARE_CPPNODE( _np_exp_timewise ) STATE_VAR( std::vector>, s_elem ); STATE_VAR( PyShape, s_shp{} ); - STATE_VAR( bool, s_first{true} ); + STATE_VAR( bool, s_first{ true } ); TS_OUTPUT( PyObjectPtr ); - INIT_CPPNODE( _np_exp_timewise ) { } + INIT_CPPNODE( _np_exp_timewise ) {} INVOKE() { @@ -941,14 +944,15 @@ DECLARE_CPPNODE( _np_exp_timewise ) if( csp.ticked( sampler ) && csp.ticked( x ) ) { - PyObject* arr = x.lastValue().get(); + PyObject * arr = x.lastValue().get(); if( unlikely( s_first ) ) { s_shp = PyShape( arr ); s_elem.reserve( s_shp.m_n ); for( int64_t j = 0; j < s_shp.m_n; j++ ) { - s_elem.emplace_back( DataValidator( min_data_points, true, halflife, now() - TimeDelta::fromMicroseconds( 1 ) ) ); + s_elem.emplace_back( + DataValidator( min_data_points, true, halflife, now() - TimeDelta::fromMicroseconds( 1 ) ) ); } s_first = false; } @@ -980,11 +984,11 @@ DECLARE_CPPNODE( _np_matrix_compute ) STATE_VAR( std::vector>, s_elem ); STATE_VAR( PyShape, s_shp{} ); - STATE_VAR( bool, s_first{true} ); + STATE_VAR( bool, s_first{ true } ); TS_OUTPUT( PyObjectPtr ); - INIT_CPPNODE( _np_matrix_compute ) { } + INIT_CPPNODE( _np_matrix_compute ) {} INVOKE() { @@ -998,14 +1002,16 @@ DECLARE_CPPNODE( _np_matrix_compute ) const std::vector & add_x = additions.lastValue(); if( unlikely( s_first ) ) { - PyObject* arr = add_x[0].get(); - if( PyArray_NDIM( (PyArrayObject * ) arr ) != 1 ) - CSP_THROW( ValueError, "Covariance or correlation matrix called on an array of more than 1 dimension: undefined" ); + PyObject * arr = add_x[0].get(); + if( PyArray_NDIM( (PyArrayObject *)arr ) != 1 ) + CSP_THROW( + ValueError, + "Covariance or correlation matrix called on an array of more than 1 dimension: undefined" ); // Special setup int64_t n = PyArray_Size( arr ); - s_shp = PyShape( {n,n}, n ); - s_elem.reserve( n*n ); - for( int64_t j = 0; j < n*n; j++ ) + s_shp = PyShape( { n, n }, n ); + s_elem.reserve( n * n ); + for( int64_t j = 0; j < n * n; j++ ) { s_elem.emplace_back( DataValidator( min_data_points, ignore_na, ddof ) ); } @@ -1015,14 +1021,14 @@ DECLARE_CPPNODE( _np_matrix_compute ) size_t m = add_x.size(); for( size_t i = 0; i < m; i++ ) { - PyObject* v = add_x[i].get(); + PyObject * v = add_x[i].get(); for( NumPyIterator iter1( v ); iter1; ++iter1 ) { double x = iter1.value(); for( NumPyIterator iter2( v ); iter2; ++iter2 ) { double y = iter2.value(); - s_elem[iter1.index()*s_shp.m_n + iter2.index()].add( x, y ); + s_elem[iter1.index() * s_shp.m_n + iter2.index()].add( x, y ); } } } @@ -1035,14 +1041,14 @@ DECLARE_CPPNODE( _np_matrix_compute ) size_t m = rem_x.size(); for( size_t i = 0; i < m; i++ ) { - PyObject* v = rem_x[i].get(); + PyObject * v = rem_x[i].get(); for( NumPyIterator iter1( v ); iter1; ++iter1 ) { double x = iter1.value(); for( NumPyIterator iter2( v ); iter2; ++iter2 ) { double y = iter2.value(); - s_elem[iter1.index()*s_shp.m_n + iter2.index()].remove( x, y ); + s_elem[iter1.index() * s_shp.m_n + iter2.index()].remove( x, y ); } } } @@ -1055,7 +1061,7 @@ DECLARE_CPPNODE( _np_matrix_compute ) } }; -EXPORT_TEMPLATE_CPPNODE( _np_cov_matrix, _np_matrix_compute ); +EXPORT_TEMPLATE_CPPNODE( _np_cov_matrix, _np_matrix_compute ); EXPORT_TEMPLATE_CPPNODE( _np_corr_matrix, _np_matrix_compute ); template @@ -1073,11 +1079,11 @@ DECLARE_CPPNODE( _np_weighted_matrix_compute ) STATE_VAR( std::vector>, s_elem ); STATE_VAR( PyShape, s_shp{} ); - STATE_VAR( bool, s_first{true} ); + STATE_VAR( bool, s_first{ true } ); TS_OUTPUT( PyObjectPtr ); - INIT_CPPNODE( _np_weighted_matrix_compute ) { } + INIT_CPPNODE( _np_weighted_matrix_compute ) {} INVOKE() { @@ -1089,17 +1095,17 @@ DECLARE_CPPNODE( _np_weighted_matrix_compute ) if( csp.ticked( x_add ) ) { const std::vector & add_x = x_add.lastValue(); - const std::vector & add_w = w_add.lastValue(); + const std::vector & add_w = w_add.lastValue(); if( unlikely( s_first ) ) { - PyObject* arr = add_x[0].get(); - if( PyArray_NDIM( (PyArrayObject * ) arr ) != 1 ) + PyObject * arr = add_x[0].get(); + if( PyArray_NDIM( (PyArrayObject *)arr ) != 1 ) CSP_THROW( ValueError, "Covariance matrix called on an array of more than 1 dimension: undefined" ); // Special setup int64_t n = PyArray_Size( arr ); - s_shp = PyShape( {n,n}, n ); - s_elem.reserve( n*n ); - for( int64_t j = 0; j < n*n; j++ ) + s_shp = PyShape( { n, n }, n ); + s_elem.reserve( n * n ); + for( int64_t j = 0; j < n * n; j++ ) { s_elem.emplace_back( DataValidator( min_data_points, ignore_na, ddof ) ); } @@ -1109,15 +1115,15 @@ DECLARE_CPPNODE( _np_weighted_matrix_compute ) size_t m = add_x.size(); for( size_t i = 0; i < m; i++ ) { - double w = add_w[i]; - PyObject* v = add_x[i].get(); + double w = add_w[i]; + PyObject * v = add_x[i].get(); for( NumPyIterator iter1( v ); iter1; ++iter1 ) { double x = iter1.value(); for( NumPyIterator iter2( v ); iter2; ++iter2 ) { double y = iter2.value(); - s_elem[iter1.index()*s_shp.m_n + iter2.index()].add( x, y, w ); + s_elem[iter1.index() * s_shp.m_n + iter2.index()].add( x, y, w ); } } } @@ -1126,20 +1132,20 @@ DECLARE_CPPNODE( _np_weighted_matrix_compute ) if( csp.ticked( x_rem ) ) { const std::vector & rem_x = x_rem.lastValue(); - const std::vector & rem_w = w_rem.lastValue(); + const std::vector & rem_w = w_rem.lastValue(); // Iterate on each element of the array size_t m = rem_x.size(); for( size_t i = 0; i < m; i++ ) { - double w = rem_w[i]; - PyObject* v = rem_x[i].get(); + double w = rem_w[i]; + PyObject * v = rem_x[i].get(); for( NumPyIterator iter1( v ); iter1; ++iter1 ) { double x = iter1.value(); for( NumPyIterator iter2( v ); iter2; ++iter2 ) { double y = iter2.value(); - s_elem[iter1.index()*s_shp.m_n + iter2.index()].remove( x, y, w ); + s_elem[iter1.index() * s_shp.m_n + iter2.index()].remove( x, y, w ); } } } @@ -1152,7 +1158,7 @@ DECLARE_CPPNODE( _np_weighted_matrix_compute ) } }; -EXPORT_TEMPLATE_CPPNODE( _np_weighted_cov_matrix, _np_weighted_matrix_compute ); +EXPORT_TEMPLATE_CPPNODE( _np_weighted_cov_matrix, _np_weighted_matrix_compute ); EXPORT_TEMPLATE_CPPNODE( _np_weighted_corr_matrix, _np_weighted_matrix_compute ); // trivariate @@ -1174,14 +1180,13 @@ DECLARE_CPPNODE( _np_trivariate ) STATE_VAR( std::vector>, s_elem{} ); STATE_VAR( PyShape, s_shp{} ); - STATE_VAR( bool, s_first{true} ); + STATE_VAR( bool, s_first{ true } ); TS_OUTPUT( PyObjectPtr ); - INIT_CPPNODE( _np_trivariate ) { } + INIT_CPPNODE( _np_trivariate ) {} public: - INVOKE() { if( csp.ticked( reset ) ) @@ -1191,14 +1196,14 @@ DECLARE_CPPNODE( _np_trivariate ) } if( csp.ticked( x_add ) ) { - const std::vector & add_x = x_add.lastValue(); - const std::vector & add_y = y_add.lastValue(); + const std::vector & add_x = x_add.lastValue(); + const std::vector & add_y = y_add.lastValue(); const std::vector & weights = w_add.lastValue(); if( unlikely( s_first ) ) { - PyObject* arr = add_x[0].get(); - s_shp = PyShape( arr ); + PyObject * arr = add_x[0].get(); + s_shp = PyShape( arr ); s_elem.reserve( s_shp.m_n ); for( int64_t j = 0; j < s_shp.m_n; j++ ) s_elem.emplace_back( DataValidator( min_data_points, ignore_na, arg ) ); @@ -1208,31 +1213,33 @@ DECLARE_CPPNODE( _np_trivariate ) size_t m = add_x.size(); for( size_t i = 0; i < m; i++ ) { - PyObject* x = add_x[i].get(); - PyObject* y = add_y[i].get(); - PyObject* w = weights[i].get(); + PyObject * x = add_x[i].get(); + PyObject * y = add_y[i].get(); + PyObject * w = weights[i].get(); s_shp.validateShape( y ); s_shp.validateShape( w ); - for( NumPyIterator x_iter = {x}, y_iter = {y}, w_iter = {w}; x_iter && y_iter && w_iter; ++x_iter, ++w_iter, ++y_iter ) + for( NumPyIterator x_iter = { x }, y_iter = { y }, w_iter = { w }; x_iter && y_iter && w_iter; + ++x_iter, ++w_iter, ++y_iter ) s_elem[x_iter.index()].add( x_iter.value(), y_iter.value(), w_iter.value() ); } } if( csp.ticked( x_rem ) ) { - const std::vector & rem_x = x_rem.lastValue(); - const std::vector & rem_y = y_rem.lastValue(); + const std::vector & rem_x = x_rem.lastValue(); + const std::vector & rem_y = y_rem.lastValue(); const std::vector & weights = w_rem.lastValue(); size_t m = rem_x.size(); for( size_t i = 0; i < m; i++ ) { - PyObject* x = rem_x[i].get(); - PyObject* y = rem_y[i].get(); - PyObject* w = weights[i].get(); + PyObject * x = rem_x[i].get(); + PyObject * y = rem_y[i].get(); + PyObject * w = weights[i].get(); - for( NumPyIterator x_iter = {x}, y_iter = {y}, w_iter = {w}; x_iter && y_iter && w_iter; ++x_iter, ++w_iter, ++y_iter ) + for( NumPyIterator x_iter = { x }, y_iter = { y }, w_iter = { w }; x_iter && y_iter && w_iter; + ++x_iter, ++w_iter, ++y_iter ) s_elem[x_iter.index()].remove( x_iter.value(), y_iter.value(), w_iter.value() ); } } @@ -1245,7 +1252,7 @@ DECLARE_CPPNODE( _np_trivariate ) }; EXPORT_TEMPLATE_CPPNODE( _np_weighted_covar, _np_trivariate ); -EXPORT_TEMPLATE_CPPNODE( _np_weighted_corr, _np_trivariate ); +EXPORT_TEMPLATE_CPPNODE( _np_weighted_corr, _np_trivariate ); DECLARE_CPPNODE( _np_arg_min_max ) { @@ -1261,10 +1268,10 @@ DECLARE_CPPNODE( _np_arg_min_max ) STATE_VAR( std::vector>, s_elem{} ); STATE_VAR( PyShape, s_shp{} ); - STATE_VAR( bool, s_first{true} ); + STATE_VAR( bool, s_first{ true } ); TS_OUTPUT( PyObjectPtr ); - INIT_CPPNODE( _np_arg_min_max ) { } + INIT_CPPNODE( _np_arg_min_max ) {} INVOKE() { @@ -1275,7 +1282,7 @@ DECLARE_CPPNODE( _np_arg_min_max ) } if( csp.ticked( x ) && csp.ticked( sampler ) ) { - PyObject* arr = x.lastValue().get(); + PyObject * arr = x.lastValue().get(); if( unlikely( s_first ) ) { s_shp = PyShape( arr ); @@ -1286,7 +1293,7 @@ DECLARE_CPPNODE( _np_arg_min_max ) } s_first = false; } - for( NumPyIterator iter = {arr}; iter; ++iter ) + for( NumPyIterator iter = { arr }; iter; ++iter ) s_elem[iter.index()].add( iter.value(), now() ); } @@ -1295,8 +1302,8 @@ DECLARE_CPPNODE( _np_arg_min_max ) size_t m = removals.lastValue().size(); for( size_t i = 0; i < m; i++ ) { - PyObject* v = removals.lastValue()[i].get(); - for( NumPyIterator iter = {v}; iter; ++iter ) + PyObject * v = removals.lastValue()[i].get(); + for( NumPyIterator iter = { v }; iter; ++iter ) s_elem[iter.index()].remove( iter.value() ); } } @@ -1306,15 +1313,16 @@ DECLARE_CPPNODE( _np_arg_min_max ) if( unlikely( s_first ) ) CSP_THROW( ValueError, NPY_SHAPE_ERROR ); PyObject * date_type = PyUnicode_FromString( " #include +#include #include #ifdef WIN32 @@ -18,52 +18,52 @@ struct Foo TEST( BasicAllocator, basic_functionality ) { std::vector allocCounts = { 32, 128, 999 }; - const size_t blockSize = 32; + const size_t blockSize = 32; for( auto allocCount : allocCounts ) { printf( "Running allocCount %lu...\n", allocCount ); bool grow = allocCount > blockSize; TypedBasicAllocator allocator( blockSize, false, grow ); - + std::vector foos; foos.resize( allocCount ); - + for( size_t i = 0; i < allocCount; ++i ) { - foos[ i ] = allocator.allocate(); + foos[i] = allocator.allocate(); ASSERT_NE( foos[i], nullptr ); - foos[i] -> a = i; - foos[i] -> b = i; - foos[i] -> c = ( char ) i; + foos[i]->a = i; + foos[i]->b = i; + foos[i]->c = (char)i; } - + if( !grow ) { ASSERT_EQ( allocator.allocate(), nullptr ); } - + for( int i = 0; i < 100000; ++i ) { int idx = random() % allocCount; if( foos[idx] ) { - //printf( "RBA to free idx: %d\n", idx ); - - ASSERT_EQ( foos[idx] -> a, idx ); - ASSERT_EQ( foos[idx] -> b, idx ); - ASSERT_EQ( foos[idx] -> c, ( char ) idx ); - + // printf( "RBA to free idx: %d\n", idx ); + + ASSERT_EQ( foos[idx]->a, idx ); + ASSERT_EQ( foos[idx]->b, idx ); + ASSERT_EQ( foos[idx]->c, (char)idx ); + allocator.free( foos[idx] ); foos[idx] = nullptr; } else { - //printf( "RBA to realloc idx: %d\n", idx ); - foos[idx] = allocator.allocate(); - foos[idx] -> a = idx; - foos[idx] -> b = idx; - foos[idx] -> c = ( char ) idx; + // printf( "RBA to realloc idx: %d\n", idx ); + foos[idx] = allocator.allocate(); + foos[idx]->a = idx; + foos[idx]->b = idx; + foos[idx]->c = (char)idx; } } } diff --git a/cpp/tests/core/test_dynamicbitset.cpp b/cpp/tests/core/test_dynamicbitset.cpp index fbcd208a..b7ed235d 100644 --- a/cpp/tests/core/test_dynamicbitset.cpp +++ b/cpp/tests/core/test_dynamicbitset.cpp @@ -1,12 +1,12 @@ -#include #include -#include +#include #include +#include #include using namespace csp; -void testIteration( const std::vector & vec, const DynamicBitSet<> & bitset ) +void testIteration( const std::vector & vec, const DynamicBitSet<> & bitset ) { size_t i = 0; for( auto it = bitset.find_first(); it != DynamicBitSet<>::npos; i++, it = bitset.find_next( it ) ) @@ -15,45 +15,47 @@ void testIteration( const std::vector & vec, const DynamicBitSet<> & bitset ASSERT_EQ( i, vec.size() ); for( auto it = bitset.find_last(); it != DynamicBitSet<>::npos; i--, it = bitset.find_prev( it ) ) - ASSERT_EQ( it, vec[i-1] ); + ASSERT_EQ( it, vec[i - 1] ); ASSERT_EQ( i, 0ul ); } TEST( DynamicBitSetTest, basic_functionality ) { - std::vector> test_cases = { std::make_pair( 4096, 400 ), std::make_pair( 200, 50 ), std::make_pair( 100, 100 ), std::make_pair( 10, 0 ) }; + std::vector> test_cases = { std::make_pair( 4096, 400 ), std::make_pair( 200, 50 ), + std::make_pair( 100, 100 ), std::make_pair( 10, 0 ) }; for( auto & test_case : test_cases ) { - auto size = test_case.first; + auto size = test_case.first; auto num_set = test_case.second; auto bitset = DynamicBitSet<>( size ); - std::unordered_set< int > vals; + std::unordered_set vals; while( vals.size() < num_set ) vals.insert( rand() % size ); - + for( auto i : vals ) bitset.set( i ); - std::vector< int > vec; - for( auto i : vals ) vec.push_back( i ); + std::vector vec; + for( auto i : vals ) + vec.push_back( i ); std::sort( vec.begin(), vec.end() ); testIteration( vec, bitset ); - std::vector< int > half_vec; - for( size_t j = 0; j < vec.size(); j+=2 ) + std::vector half_vec; + for( size_t j = 0; j < vec.size(); j += 2 ) { bitset.reset( vec[j] ); - half_vec.push_back( vec[j+1] ); + half_vec.push_back( vec[j + 1] ); } std::sort( half_vec.begin(), half_vec.end() ); testIteration( half_vec, bitset ); - for( size_t j = 1; j < vec.size(); j+=2 ) + for( size_t j = 1; j < vec.size(); j += 2 ) bitset.reset( vec[j] ); ASSERT_EQ( bitset.find_first(), DynamicBitSet<>::npos ); diff --git a/cpp/tests/core/test_enum.cpp b/cpp/tests/core/test_enum.cpp index 7c3b8524..f010c394 100644 --- a/cpp/tests/core/test_enum.cpp +++ b/cpp/tests/core/test_enum.cpp @@ -1,6 +1,6 @@ #include -#include #include +#include using namespace csp; @@ -22,13 +22,7 @@ struct TestEnumTraits using TestEnum = Enum; -INIT_CSP_ENUM(TestEnum, - "UNKNOWN", - "A", - "B", - "C", - "F" -); +INIT_CSP_ENUM( TestEnum, "UNKNOWN", "A", "B", "C", "F" ); TEST( EnumTest, basic_functionality ) { @@ -42,7 +36,6 @@ TEST( EnumTest, basic_functionality ) ASSERT_NE( TestEnum( TestEnum::B ), TestEnum::A ); ASSERT_NE( TestEnum::A, TestEnum( 2 ) ); - ASSERT_EQ( TestEnum( 1 ), TestEnum::A ); ASSERT_EQ( TestEnum( 1 ).asString(), "A" ); ASSERT_EQ( TestEnum( 2 ).asString(), "B" ); @@ -56,7 +49,7 @@ TEST( EnumTest, basic_functionality ) std::stringstream oss; oss << TestEnum( "UNKNOWN" ); - ASSERT_EQ( oss.str(), "UNKNOWN"); + ASSERT_EQ( oss.str(), "UNKNOWN" ); ASSERT_THROW( TestEnum( "FOO" ), ValueError ); ASSERT_THROW( TestEnum( 23 ), ValueError ); @@ -67,16 +60,16 @@ TEST( EnumTest, iteration ) std::set out; for( auto it = TestEnum::begin(); it != TestEnum::end(); ++it ) { - out.insert( (*it).asString() ); + out.insert( ( *it ).asString() ); } - decltype( out ) comp{ "UNKNOWN", "A", "B" ,"C", "F" }; + decltype( out ) comp{ "UNKNOWN", "A", "B", "C", "F" }; ASSERT_EQ( out, comp ); } TEST( EnumTest, test_external_init ) { - //was a problem with static linking + // was a problem with static linking ASSERT_EQ( PushMode( "LAST_VALUE" ), PushMode::LAST_VALUE ); - ASSERT_EQ( PushMode( PushMode::LAST_VALUE ).asString(), "LAST_VALUE" ); + ASSERT_EQ( PushMode( PushMode::LAST_VALUE ).asString(), "LAST_VALUE" ); } diff --git a/cpp/tests/core/test_platform.cpp b/cpp/tests/core/test_platform.cpp index 42bb6993..2397cea8 100644 --- a/cpp/tests/core/test_platform.cpp +++ b/cpp/tests/core/test_platform.cpp @@ -1,23 +1,23 @@ -#include #include +#include TEST( Platform, clz ) { uint8_t x = 1; for( int i = 7; i >= 0; --i, x <<= 1 ) - ASSERT_EQ( clz( x ), ( uint8_t ) i ); + ASSERT_EQ( clz( x ), (uint8_t)i ); uint16_t y = 1; for( int i = 15; i >= 0; --i, y <<= 1 ) - ASSERT_EQ( clz( y ), ( uint16_t ) i ); + ASSERT_EQ( clz( y ), (uint16_t)i ); uint32_t z = 1; for( int i = 31; i >= 0; --i, z <<= 1 ) - ASSERT_EQ( clz( z ), ( uint32_t ) i ); + ASSERT_EQ( clz( z ), (uint32_t)i ); uint64_t w = 1; for( int i = 63; i >= 0; --i, w <<= 1 ) - ASSERT_EQ( clz( w ), ( uint64_t ) i ); + ASSERT_EQ( clz( w ), (uint64_t)i ); } TEST( Platform, ffs ) diff --git a/cpp/tests/core/test_srmwlockfreequeue.cpp b/cpp/tests/core/test_srmwlockfreequeue.cpp index e230b0c8..3cd2a0d8 100644 --- a/cpp/tests/core/test_srmwlockfreequeue.cpp +++ b/cpp/tests/core/test_srmwlockfreequeue.cpp @@ -1,18 +1,21 @@ -#include #include #include +#include using namespace csp; struct TestEvent { - TestEvent( int v_ ) : v( v_ ), next( nullptr ) {} + TestEvent( int v_ ) + : v( v_ ) + , next( nullptr ) + { + } int v; TestEvent * next; }; - TEST( SRMWLockFreeQueueNoThreads, basic_functionality ) { SRMWLockFreeQueue q; @@ -23,9 +26,9 @@ TEST( SRMWLockFreeQueueNoThreads, basic_functionality ) q.push( new TestEvent( n * 100 + i ) ); TestEvent * evt; - while( ( evt = ( TestEvent * ) q.pop() ) ) + while( ( evt = (TestEvent *)q.pop() ) ) { - ASSERT_EQ( evt -> v, x++ ); + ASSERT_EQ( evt->v, x++ ); delete evt; } } @@ -36,7 +39,7 @@ int64_t pushEvents( SRMWLockFreeQueue * q, int tickCount, int id ) int64_t sum = 0; for( int i = 0; i < tickCount; ++i ) { - q -> push( new TestEvent( tickCount * id + i ) ); + q->push( new TestEvent( tickCount * id + i ) ); sum += tickCount * id + i; } @@ -56,9 +59,8 @@ int64_t pushBatchEvents( SRMWLockFreeQueue * q, int tickCount, int ba sum += tickCount * id + i + j; } - q -> push( batch ); + q->push( batch ); } - } return sum; @@ -66,24 +68,24 @@ int64_t pushBatchEvents( SRMWLockFreeQueue * q, int tickCount, int ba TEST( SRMWLockFreeQueueWThreads, basic_functionality ) { - //test multiple writers competing + // test multiple writers competing int num_threads = 16; - int tickCount = 1000000; + int tickCount = 1000000; SRMWLockFreeQueue q; - std::vector > res; + std::vector> res; for( int i = 0; i < num_threads; ++i ) res.push_back( std::async( std::launch::async, pushEvents, &q, tickCount, i ) ); - int64_t c = 0; + int64_t c = 0; int64_t sum = 0; while( c < num_threads * tickCount ) { - TestEvent * e = ( TestEvent * ) q.pop(); + TestEvent * e = (TestEvent *)q.pop(); if( e ) { ++c; - sum += e -> v; + sum += e->v; delete e; } } @@ -93,29 +95,28 @@ TEST( SRMWLockFreeQueueWThreads, basic_functionality ) expected += r.get(); ASSERT_EQ( sum, expected ); - } TEST( SRMWLockFreeQueueWThreads, batch_functionality ) { - //test multiple writers competing + // test multiple writers competing int num_threads = 16; - int tickCount = 1000000; + int tickCount = 1000000; SRMWLockFreeQueue q; - std::vector > res; + std::vector> res; for( int i = 0; i < num_threads; ++i ) res.push_back( std::async( std::launch::async, pushBatchEvents, &q, tickCount, 20, i ) ); - int64_t c = 0; + int64_t c = 0; int64_t sum = 0; while( c < num_threads * tickCount ) { - TestEvent * e = ( TestEvent * ) q.pop(); + TestEvent * e = (TestEvent *)q.pop(); if( e ) { ++c; - sum += e -> v; + sum += e->v; delete e; } } @@ -125,17 +126,16 @@ TEST( SRMWLockFreeQueueWThreads, batch_functionality ) expected += r.get(); ASSERT_EQ( sum, expected ); - } TEST( SRMWLockFreeQueueNoThreads, test_empty ) { SRMWLockFreeQueue q; for( int i = 0; i < 100; ++i ) - q.push( new TestEvent( i ) ); - ASSERT_FALSE(q.empty()); + q.push( new TestEvent( i ) ); + ASSERT_FALSE( q.empty() ); delete q.pop(); - ASSERT_FALSE(q.empty()); + ASSERT_FALSE( q.empty() ); while( !q.empty() ) delete q.pop(); @@ -146,20 +146,22 @@ void testBlockingWait( TimeDelta maxWait ) SRMWLockFreeQueue q( true ); int nEvents = 1000; - auto result = std::async( std::launch::async, [&](){ - int eventsRecvd = 0; - int total = 0; - while( eventsRecvd < nEvents ) - { - for( auto * e = q.pop( maxWait ); e != nullptr; e = q.pop( maxWait ) ) - { - ++eventsRecvd; - total += e -> v; - delete e; - } - } - return total; - }); + auto result = std::async( std::launch::async, + [&]() + { + int eventsRecvd = 0; + int total = 0; + while( eventsRecvd < nEvents ) + { + for( auto * e = q.pop( maxWait ); e != nullptr; e = q.pop( maxWait ) ) + { + ++eventsRecvd; + total += e->v; + delete e; + } + } + return total; + } ); int expectedResult = 0; for( int i = 0; i < nEvents; ++i ) diff --git a/cpp/tests/core/test_tagged_pointer_union.cpp b/cpp/tests/core/test_tagged_pointer_union.cpp index 35dff84b..57c6c794 100644 --- a/cpp/tests/core/test_tagged_pointer_union.cpp +++ b/cpp/tests/core/test_tagged_pointer_union.cpp @@ -1,19 +1,19 @@ -#include #include +#include #include using namespace csp; TEST( TaggedPointerUnion, test_basic_functionality ) { - using TP2 = TaggedPointerUnion; + using TP2 = TaggedPointerUnion; TP2 tp2; - + ASSERT_EQ( TP2::NUM_TAGS, 2u ); ASSERT_EQ( TP2::TAG_BITS, 2u ); ASSERT_EQ( TP2::TAG_MASK, 3u ); - ASSERT_EQ( TP2::typeBit(), 0u ); + ASSERT_EQ( TP2::typeBit(), 0u ); ASSERT_EQ( TP2::typeBit(), 1u ); ASSERT_EQ( tp2.raw(), nullptr ); diff --git a/cpp/tests/core/test_time.cpp b/cpp/tests/core/test_time.cpp index a50bf15d..aac524aa 100644 --- a/cpp/tests/core/test_time.cpp +++ b/cpp/tests/core/test_time.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include using namespace csp; @@ -7,7 +7,7 @@ using namespace csp; TEST( DateTimeTest, test_basic_functionality ) { DateTime t1 = DateTime::now(); - std::this_thread::sleep_for( std::chrono::milliseconds(50) ); + std::this_thread::sleep_for( std::chrono::milliseconds( 50 ) ); EXPECT_NE( t1, DateTime::now() ); DateTime d1( 2017, 1, 1, 13, 1, 17, 123456789 ); @@ -23,7 +23,7 @@ TEST( DateTimeTest, test_basic_functionality ) EXPECT_TRUE( d1 - TimeDelta::fromNanoseconds( 1 ) <= d2 ); EXPECT_FALSE( d1 - TimeDelta::fromNanoseconds( 1 ) < d2 ); - + d1 = DateTime( 2017, 1, 1, 1, 1 ); d2 = DateTime( 2017, 1, 2, 2, 2 ); @@ -39,22 +39,27 @@ TEST( DateTimeTest, test_basic_functionality ) EXPECT_EQ( DateTime( Date( 2017, 1, 2 ), Time( 15, 14, 14, 123456 ) ), DateTime( 2017, 1, 2, 15, 14, 14, 123456 ) ); - EXPECT_EQ( DateTime( 2017, 1, 1, 1, 1, 1, 123456789 ).roundDown( TimeDelta::fromSeconds( 1 ) ), DateTime( 2017, 1, 1, 1, 1, 1 ) ); + EXPECT_EQ( DateTime( 2017, 1, 1, 1, 1, 1, 123456789 ).roundDown( TimeDelta::fromSeconds( 1 ) ), + DateTime( 2017, 1, 1, 1, 1, 1 ) ); - EXPECT_EQ( DateTime( 2017, 1, 1, 1, 1, 1, 823456789 ).roundDown( TimeDelta::fromMilliseconds( 500 ) ), DateTime( 2017, 1, 1, 1, 1, 1, 500000000 ) ); - EXPECT_EQ( DateTime( 2017, 1, 1, 1, 1, 1, 123456789 ).roundDown( TimeDelta::fromMilliseconds( 500 ) ), DateTime( 2017, 1, 1, 1, 1, 1 ) ); - EXPECT_EQ( DateTime( 2017, 1, 1, 1, 1, 1, 123456789 ).roundDown( TimeDelta::fromSeconds( 30 ) ), DateTime( 2017, 1, 1, 1, 1, 0 ) ); - EXPECT_EQ( DateTime( 2017, 1, 1, 1, 1, 31, 123456789 ).roundDown( TimeDelta::fromSeconds( 30 ) ), DateTime( 2017, 1, 1, 1, 1, 30 ) ); + EXPECT_EQ( DateTime( 2017, 1, 1, 1, 1, 1, 823456789 ).roundDown( TimeDelta::fromMilliseconds( 500 ) ), + DateTime( 2017, 1, 1, 1, 1, 1, 500000000 ) ); + EXPECT_EQ( DateTime( 2017, 1, 1, 1, 1, 1, 123456789 ).roundDown( TimeDelta::fromMilliseconds( 500 ) ), + DateTime( 2017, 1, 1, 1, 1, 1 ) ); + EXPECT_EQ( DateTime( 2017, 1, 1, 1, 1, 1, 123456789 ).roundDown( TimeDelta::fromSeconds( 30 ) ), + DateTime( 2017, 1, 1, 1, 1, 0 ) ); + EXPECT_EQ( DateTime( 2017, 1, 1, 1, 1, 31, 123456789 ).roundDown( TimeDelta::fromSeconds( 30 ) ), + DateTime( 2017, 1, 1, 1, 1, 30 ) ); ASSERT_EQ( DateTime( 2018, 1, 1, 14, 1, 2, 123456789 ).asString(), "20180101 14:01:02.123456789" ); ASSERT_EQ( DateTime::MIN_VALUE().asString(), "min" ); ASSERT_EQ( DateTime::MAX_VALUE().asString(), "max" ); - ASSERT_EQ( DateTime::NONE().asString(), "none" ); + ASSERT_EQ( DateTime::NONE().asString(), "none" ); ASSERT_TRUE( DateTime().isNone() ); } - + TEST( TimeDeltaTest, test_basic_functionality ) { TimeDelta delta1 = TimeDelta::fromSeconds( 86400 * 3 + 3600 * 4 + 5 * 60 + 15 ); @@ -64,7 +69,8 @@ TEST( TimeDeltaTest, test_basic_functionality ) EXPECT_EQ( delta1.seconds(), 15 ); EXPECT_EQ( delta1.nanoseconds(), 0 ); - TimeDelta delta2 = TimeDelta::fromNanoseconds( (86400 * 3 + 3600 * 4 + 5 * 60 + 15 ) * NANOS_PER_SECOND + 123456789 ); + TimeDelta delta2 + = TimeDelta::fromNanoseconds( ( 86400 * 3 + 3600 * 4 + 5 * 60 + 15 ) * NANOS_PER_SECOND + 123456789 ); EXPECT_EQ( delta2.days(), 3 ); EXPECT_EQ( delta2.hours(), 4 ); EXPECT_EQ( delta2.minutes(), 5 ); @@ -94,13 +100,15 @@ TEST( TimeDeltaTest, test_basic_functionality ) ASSERT_EQ( delta1.asString(), "3 days 04:05:15" ); ASSERT_EQ( delta2.asString(), "3 days 04:05:15.123456789" ); - ASSERT_EQ( TimeDelta::fromString( "04:05:15" ), TimeDelta::fromSeconds( 4 * 3600 + 5 * 60 + 15 ) ); - ASSERT_EQ( TimeDelta::fromString( "05:15.123" ), TimeDelta::fromSeconds( 5 * 60 + 15 ) + TimeDelta::fromMilliseconds( 123 ) ); - ASSERT_EQ( TimeDelta::fromString( "2.123456" ), TimeDelta::fromSeconds( 2 ) + TimeDelta::fromMicroseconds( 123456 ) ); + ASSERT_EQ( TimeDelta::fromString( "04:05:15" ), TimeDelta::fromSeconds( 4 * 3600 + 5 * 60 + 15 ) ); + ASSERT_EQ( TimeDelta::fromString( "05:15.123" ), + TimeDelta::fromSeconds( 5 * 60 + 15 ) + TimeDelta::fromMilliseconds( 123 ) ); + ASSERT_EQ( TimeDelta::fromString( "2.123456" ), + TimeDelta::fromSeconds( 2 ) + TimeDelta::fromMicroseconds( 123456 ) ); - ASSERT_EQ( TimeDelta::fromSeconds( 3 * 60 + 15 ).asString(), "00:03:15" ); - ASSERT_EQ( TimeDelta::fromHours( 25 ).asString(), "1 day 01:00:00" ); - ASSERT_EQ( TimeDelta::fromDays( 25 ).asString(), "25 days 00:00:00" ); + ASSERT_EQ( TimeDelta::fromSeconds( 3 * 60 + 15 ).asString(), "00:03:15" ); + ASSERT_EQ( TimeDelta::fromHours( 25 ).asString(), "1 day 01:00:00" ); + ASSERT_EQ( TimeDelta::fromDays( 25 ).asString(), "25 days 00:00:00" ); { TimeDelta t1 = TimeDelta::fromSeconds( 60 ); @@ -118,7 +126,7 @@ TEST( TimeDeltaTest, test_basic_functionality ) { TimeDelta td = TimeDelta::fromMicroseconds( 123 ); - td = -td; + td = -td; ASSERT_EQ( td, TimeDelta::fromMicroseconds( -123 ) ); td = -td; ASSERT_EQ( td, TimeDelta::fromMicroseconds( 123 ) ); @@ -132,57 +140,56 @@ TEST( DateTimeEx, test_basic_functionality ) DateTime dt( 2017, 3, 7, 13, 2, 17, 123456789 ); DateTimeEx ex( dt ); - ASSERT_EQ( ex.year(), 2017 ); - ASSERT_EQ( ex.month(), 3 ); - ASSERT_EQ( ex.day(), 7 ); - ASSERT_EQ( ex.hour(), 13 ); + ASSERT_EQ( ex.year(), 2017 ); + ASSERT_EQ( ex.month(), 3 ); + ASSERT_EQ( ex.day(), 7 ); + ASSERT_EQ( ex.hour(), 13 ); ASSERT_EQ( ex.minute(), 2 ); ASSERT_EQ( ex.second(), 17 ); ASSERT_EQ( ex.milliseconds(), 123 ); ASSERT_EQ( ex.microseconds(), 123456 ); - ASSERT_EQ( ex.nanoseconds(), 123456789 ); + ASSERT_EQ( ex.nanoseconds(), 123456789 ); - - //negative ticks, pre-1970 + // negative ticks, pre-1970 dt = DateTime( 1969, 12, 31, 23, 59, 59, 123456789 ); DateTimeEx ex2( dt ); - ASSERT_EQ( ex2.year(), 1969 ); - ASSERT_EQ( ex2.month(), 12 ); - ASSERT_EQ( ex2.day(), 31 ); - ASSERT_EQ( ex2.hour(), 23 ); + ASSERT_EQ( ex2.year(), 1969 ); + ASSERT_EQ( ex2.month(), 12 ); + ASSERT_EQ( ex2.day(), 31 ); + ASSERT_EQ( ex2.hour(), 23 ); ASSERT_EQ( ex2.minute(), 59 ); ASSERT_EQ( ex2.second(), 59 ); ASSERT_EQ( ex2.milliseconds(), 123 ); ASSERT_EQ( ex2.microseconds(), 123456 ); - ASSERT_EQ( ex2.nanoseconds(), 123456789 ); + ASSERT_EQ( ex2.nanoseconds(), 123456789 ); - //Only linux supports time before EPOCH + // Only linux supports time before EPOCH #ifdef __linux__ dt = DateTime( 1888, 11, 15, 23, 15, 59, 999999999 ); DateTimeEx ex3( dt ); - ASSERT_EQ( ex3.year(), 1888 ); - ASSERT_EQ( ex3.month(), 11 ); - ASSERT_EQ( ex3.day(), 15 ); - ASSERT_EQ( ex3.hour(), 23 ); + ASSERT_EQ( ex3.year(), 1888 ); + ASSERT_EQ( ex3.month(), 11 ); + ASSERT_EQ( ex3.day(), 15 ); + ASSERT_EQ( ex3.hour(), 23 ); ASSERT_EQ( ex3.minute(), 15 ); ASSERT_EQ( ex3.second(), 59 ); ASSERT_EQ( ex3.milliseconds(), 999 ); ASSERT_EQ( ex3.microseconds(), 999999 ); - ASSERT_EQ( ex3.nanoseconds(), 999999999 ); + ASSERT_EQ( ex3.nanoseconds(), 999999999 ); #endif } TEST( TimeTest, test_basic_functionality ) { Time t( 14, 1, 2, 123456789 ); - ASSERT_EQ( t.hour(), 14 ); - ASSERT_EQ( t.minute(), 1 ); - ASSERT_EQ( t.second(), 2 ); + ASSERT_EQ( t.hour(), 14 ); + ASSERT_EQ( t.minute(), 1 ); + ASSERT_EQ( t.second(), 2 ); ASSERT_EQ( t.nanosecond(), 123456789 ); Time t2( 14, 1, 2, 123456790 ); @@ -190,14 +197,14 @@ TEST( TimeTest, test_basic_functionality ) ASSERT_TRUE( t < t2 ); ASSERT_FALSE( t == t2 ); ASSERT_TRUE( t != t2 ); - + ASSERT_EQ( t.asString(), "14:01:02.123456789" ); ASSERT_EQ( Time::fromString( "14:01:02.123456789" ), t ); ASSERT_EQ( Time::fromString( "14:01:02.1234" ), Time( 14, 1, 2, 123400000 ) ); - ASSERT_EQ( Time::fromString( "14:01:02.12" ), Time( 14, 1, 2, 120000000 ) ); - ASSERT_EQ( Time::fromString( "14:01:02" ), Time( 14, 1, 2 ) ); - ASSERT_EQ( Time::fromString( "14:01" ), Time( 14, 1, 0 ) ); - ASSERT_EQ( Time::fromString( "14" ), Time( 14, 0, 0 ) ); + ASSERT_EQ( Time::fromString( "14:01:02.12" ), Time( 14, 1, 2, 120000000 ) ); + ASSERT_EQ( Time::fromString( "14:01:02" ), Time( 14, 1, 2 ) ); + ASSERT_EQ( Time::fromString( "14:01" ), Time( 14, 1, 0 ) ); + ASSERT_EQ( Time::fromString( "14" ), Time( 14, 0, 0 ) ); DateTime dt( 2017, 1, 1, 14, 1, 2, 123456789 ); ASSERT_EQ( dt.time(), t ); @@ -207,7 +214,7 @@ TEST( TimeTest, test_basic_functionality ) ASSERT_EQ( dt3, DateTime( 2017, 1, 1, 14, 1, 2, 123456789 ) ); - //math + // math Time t3( 14, 30, 0 ); ASSERT_EQ( t3 + TimeDelta::fromSeconds( 91 ), Time( 14, 31, 31 ) ); ASSERT_EQ( t3 - TimeDelta::fromSeconds( 91 ), Time( 14, 28, 29 ) ); @@ -220,7 +227,7 @@ TEST( TimeTest, test_basic_functionality ) ASSERT_EQ( Time( 14, 31, 31 ) - Time( 14, 30, 0 ), TimeDelta::fromSeconds( 91 ) ); //"daylight savings" ( not that there is any, its UTC - dt2 = DateTime( 2017, 5, 1, 1, 1, 1); + dt2 = DateTime( 2017, 5, 1, 1, 1, 1 ); dt3 = dt2.withTime( t ); ASSERT_EQ( dt3, DateTime( 2017, 5, 1, 14, 1, 2, 123456789 ) ); @@ -232,11 +239,11 @@ TEST( DateTest, test_strfortime ) Date d( 2017, 2, 3 ); std::string format = "/tmp/univ_%Y%m%d.txt"; char fileName[1024]; - d.strftime( fileName, sizeof( fileName ), format.c_str()); + d.strftime( fileName, sizeof( fileName ), format.c_str() ); ASSERT_STREQ( "/tmp/univ_20170203.txt", fileName ); format = "/data_root/univ/%Y/%m"; - d.strftime( fileName, sizeof( fileName ), format.c_str()); + d.strftime( fileName, sizeof( fileName ), format.c_str() ); ASSERT_STREQ( "/data_root/univ/2017/02", fileName ); ASSERT_TRUE( Date().isNone() ); @@ -245,9 +252,9 @@ TEST( DateTest, test_strfortime ) TEST( DateTest, test_basic_functionality ) { Date d( 2017, 2, 3 ); - ASSERT_EQ( d.year(), 2017 ); + ASSERT_EQ( d.year(), 2017 ); ASSERT_EQ( d.month(), 2 ); - ASSERT_EQ( d.day(), 3 ); + ASSERT_EQ( d.day(), 3 ); ASSERT_EQ( d, Date( 2017, 2, 3 ) ); ASSERT_NE( d, Date( 2017, 2, 4 ) ); @@ -270,21 +277,21 @@ TEST( DateTest, test_basic_functionality ) DateTime dt( 2017, 2, 3, 15, 16, 7 ); ASSERT_EQ( dt.date(), d ); - //date math tests + // date math tests Date d1( 2017, 1, 1 ); Date d2( 2017, 2, 1 ); ASSERT_EQ( d2 - d1, TimeDelta::fromDays( 31 ) ); ASSERT_EQ( d1 - d2, TimeDelta::fromDays( -31 ) ); ASSERT_EQ( d1 + TimeDelta::fromDays( 31 ), d2 ); ASSERT_EQ( d2 - TimeDelta::fromDays( 31 ), d1 ); - + Date d3 = d1; d3 += TimeDelta::fromDays( 31 ); ASSERT_EQ( d3, d2 ); d3 -= TimeDelta::fromDays( 31 ); ASSERT_EQ( d3, d1 ); - //weekday + // weekday { Date base( 2017, 1, 1 ); for( int days = 0; days < 7; ++days ) @@ -299,11 +306,10 @@ TEST( DateTest, test_basic_functionality ) TEST( sleep, basic_functionality ) { TimeDelta waittime = TimeDelta::fromSeconds( 1 ); - DateTime t1 = DateTime::now(); + DateTime t1 = DateTime::now(); csp::sleep( waittime ); DateTime t2 = DateTime::now(); ASSERT_GT( t2 - t1, waittime ); } - diff --git a/cpp/tests/engine/test_dictionary.cpp b/cpp/tests/engine/test_dictionary.cpp index 94609b6d..dc684c65 100644 --- a/cpp/tests/engine/test_dictionary.cpp +++ b/cpp/tests/engine/test_dictionary.cpp @@ -1,5 +1,5 @@ -#include #include +#include using namespace csp; @@ -7,13 +7,13 @@ namespace csp { DialectGenericType::DialectGenericType() {} DialectGenericType::~DialectGenericType() {} -DialectGenericType::DialectGenericType( const DialectGenericType &rhs ) {} -DialectGenericType::DialectGenericType( DialectGenericType &&rhs ) {} -DialectGenericType &DialectGenericType::operator=( const DialectGenericType &rhs ) { return *this; } -DialectGenericType &DialectGenericType::operator=( DialectGenericType &&rhs ) { return *this; } -bool DialectGenericType::operator==( const DialectGenericType &rhs ) const { return true; } +DialectGenericType::DialectGenericType( const DialectGenericType & rhs ) {} +DialectGenericType::DialectGenericType( DialectGenericType && rhs ) {} +DialectGenericType & DialectGenericType::operator=( const DialectGenericType & rhs ) { return *this; } +DialectGenericType & DialectGenericType::operator=( DialectGenericType && rhs ) { return *this; } +bool DialectGenericType::operator==( const DialectGenericType & rhs ) const { return true; } size_t DialectGenericType::hash() const { return 0; } -} +} // namespace csp TEST( Dictionary, test_basic_functionality ) { @@ -21,10 +21,10 @@ TEST( Dictionary, test_basic_functionality ) ASSERT_TRUE( d1.insert( "123", 123 ) ); ASSERT_FALSE( d1.insert( "123", 456 ) ); ASSERT_TRUE( d1.insert( "bool", true ) ); - ASSERT_TRUE( d1.insert( "int32", (int32_t ) 456 ) ); - ASSERT_TRUE( d1.insert( "uint32", (uint32_t ) 789 ) ); - ASSERT_TRUE( d1.insert( "int64", int64_t(1) << 45 ) ); - ASSERT_TRUE( d1.insert( "uint64", ( uint64_t(1) << 63 ) + 1 ) ); + ASSERT_TRUE( d1.insert( "int32", (int32_t)456 ) ); + ASSERT_TRUE( d1.insert( "uint32", (uint32_t)789 ) ); + ASSERT_TRUE( d1.insert( "int64", int64_t( 1 ) << 45 ) ); + ASSERT_TRUE( d1.insert( "uint64", ( uint64_t( 1 ) << 63 ) + 1 ) ); ASSERT_TRUE( d1.insert( "double", 123.456 ) ); ASSERT_TRUE( d1.insert( "string", std::string( "HOWDY!" ) ) ); ASSERT_TRUE( d1.insert( "string2", "HOWDY2!" ) ); @@ -35,14 +35,14 @@ TEST( Dictionary, test_basic_functionality ) ASSERT_FALSE( d1.exists( "1234" ) ); ASSERT_EQ( d1.get( "123" ), 123 ); - ASSERT_EQ( d1.get( "bool" ), true ); + ASSERT_EQ( d1.get( "bool" ), true ); ASSERT_EQ( d1.get( "int32" ), 456 ); - ASSERT_EQ( d1.get( "int64" ), int64_t(1) << 45 ); + ASSERT_EQ( d1.get( "int64" ), int64_t( 1 ) << 45 ); ASSERT_EQ( d1.get( "uint32" ), 789 ); - ASSERT_EQ( d1.get( "uint64" ), ( uint64_t(1) << 63 ) + 1 ); - ASSERT_EQ( d1.get( "double" ), 123.456 ); - ASSERT_EQ( d1.get( "time" ), DateTime( 2020, 4, 22, 10, 26 ) ); - ASSERT_EQ( d1.get( "timedelta" ), TimeDelta::fromMilliseconds( 123 ) ); + ASSERT_EQ( d1.get( "uint64" ), ( uint64_t( 1 ) << 63 ) + 1 ); + ASSERT_EQ( d1.get( "double" ), 123.456 ); + ASSERT_EQ( d1.get( "time" ), DateTime( 2020, 4, 22, 10, 26 ) ); + ASSERT_EQ( d1.get( "timedelta" ), TimeDelta::fromMilliseconds( 123 ) ); ASSERT_EQ( d1.get( "string" ), "HOWDY!" ); ASSERT_EQ( d1.get( "string2" ), "HOWDY2!" ); @@ -59,11 +59,11 @@ TEST( Dictionary, test_comp_hash ) vec.emplace_back( "a" ); vec.emplace_back( "b" ); vec.emplace_back( "c" ); - + d1.insert( "123", 0 ); d1.insert( "bool", true ); - d1.insert( "int32", (int32_t ) 456 ); - d1.insert( "int64", int64_t(1) << 45 ); + d1.insert( "int32", (int32_t)456 ); + d1.insert( "int64", int64_t( 1 ) << 45 ); d1.insert( "double", 123.456 ); d1.insert( "string", std::string( "HOWDY!" ) ); d1.insert( "string2", "HOWDY2!" ); @@ -72,18 +72,17 @@ TEST( Dictionary, test_comp_hash ) d1.insert( "vector", vec ); // When we update should be the same as if inserted at first as 123 d1.update( "123", 123 ); - ASSERT_EQ(d1.begin().key(), "123"); - + ASSERT_EQ( d1.begin().key(), "123" ); - //do d2 backwards to ensure ordering in comp / hash is good + // do d2 backwards to ensure ordering in comp / hash is good d2.insert( "vector", vec ); d2.insert( "timedelta", TimeDelta::fromMilliseconds( 123 ) ); d2.insert( "time", DateTime( 2020, 4, 22, 10, 26 ) ); d2.insert( "string2", "HOWDY2!" ); d2.insert( "string", std::string( "HOWDY!" ) ); d2.insert( "double", 123.456 ); - d2.insert( "int64", int64_t(1) << 45 ); - d2.insert( "int32", (int32_t ) 456 ); + d2.insert( "int64", int64_t( 1 ) << 45 ); + d2.insert( "int32", (int32_t)456 ); d2.insert( "bool", true ); d2.insert( "123", 123 ); @@ -112,11 +111,11 @@ TEST( Dictionary, test_type_coercion ) vec.emplace_back( "b" ); vec.emplace_back( "c" ); - d1.insert( "d", 123.456 ); - d1.insert( "i64", (int64_t) 123 ); - d1.insert( "i32", (int32_t) 456 ); - d1.insert( "u32", (uint32_t) 789 ); - d1.insert( "u64", (uint64_t) 111 ); + d1.insert( "d", 123.456 ); + d1.insert( "i64", (int64_t)123 ); + d1.insert( "i32", (int32_t)456 ); + d1.insert( "u32", (uint32_t)789 ); + d1.insert( "u64", (uint64_t)111 ); d1.insert( "vec", vec ); ASSERT_EQ( d1.get( "d" ), 123.456 ); @@ -125,7 +124,7 @@ TEST( Dictionary, test_type_coercion ) ASSERT_EQ( d1.get( "u64" ), 111.0 ); ASSERT_EQ( d1.get( "u32" ), 789.0 ); - ASSERT_EQ( d1.get( "u32" ), 789 ); + ASSERT_EQ( d1.get( "u32" ), 789 ); ASSERT_EQ( d1.get( "i32" ), 456 ); ASSERT_EQ( d1.get( "i32" ), 456 ); @@ -142,9 +141,9 @@ TEST( Dictionary, test_type_coercion ) TEST( Dictionary, test_iteration ) { Dictionary d1; - d1.insert( "d", 123.456 ); - d1.insert( "i64", (int64_t) 123 ); - d1.insert( "i32", (int32_t) 456 ); + d1.insert( "d", 123.456 ); + d1.insert( "i64", (int64_t)123 ); + d1.insert( "i32", (int32_t)456 ); d1.insert( "string", std::string( "HOWDY!" ) ); for( auto it = d1.begin(); it != d1.end(); ++it ) @@ -165,14 +164,14 @@ TEST( Dictionary, test_iteration ) TEST( Dictionary, test_composition ) { DictionaryPtr subdict = std::make_shared(); - subdict -> insert( "sub1", 123.456 ); - subdict -> insert( "sub2", "sub2" ); + subdict->insert( "sub1", 123.456 ); + subdict->insert( "sub2", "sub2" ); Dictionary d1; d1.insert( "i", 123 ); d1.insert( "sub", subdict ); ASSERT_EQ( d1.get( "i" ), 123 ); - ASSERT_EQ( d1.get( "sub" ) -> get( "sub1" ), 123.456 ); - ASSERT_EQ( d1.get( "sub" ) -> get( "sub2" ), "sub2" ); + ASSERT_EQ( d1.get( "sub" )->get( "sub1" ), 123.456 ); + ASSERT_EQ( d1.get( "sub" )->get( "sub2" ), "sub2" ); } diff --git a/cpp/tests/engine/test_engine_initial.cpp b/cpp/tests/engine/test_engine_initial.cpp index 26aee4b4..3364d60b 100644 --- a/cpp/tests/engine/test_engine_initial.cpp +++ b/cpp/tests/engine/test_engine_initial.cpp @@ -1,29 +1,29 @@ -#include -#include -#include -#include #include +#include +#include +#include +#include using namespace tcep; class AddNode : public Node { public: - AddNode( Engine * engine ) : Node( NodeDef( 2, 1 ), engine ) + AddNode( Engine * engine ) + : Node( NodeDef( 2, 1 ), engine ) { createOutput( InputId( 0 ) ); } void executeImpl() { - if( input( InputId( 0 ) ) -> valid() && - input( InputId( 1 ) ) -> valid() ) + if( input( InputId( 0 ) )->valid() && input( InputId( 1 ) )->valid() ) { - double sum = input( InputId( 0 ) ) -> lastValueTyped() + - input( InputId( 1 ) ) -> lastValueTyped(); - output( InputId( 0 ) ) -> outputTickTyped( engine() -> time(), sum ); + double sum + = input( InputId( 0 ) )->lastValueTyped() + input( InputId( 1 ) )->lastValueTyped(); + output( InputId( 0 ) )->outputTickTyped( engine()->time(), sum ); - //printf( "RBA Output from adder at time %s\n", engine() -> time().asString().c_str() ); + // printf( "RBA Output from adder at time %s\n", engine() -> time().asString().c_str() ); } } }; @@ -31,15 +31,18 @@ class AddNode : public Node class WriteNode : public Node { public: - WriteNode( Engine * engine, std::string tag ) : Node( NodeDef( 1, 0 ), engine ), - m_tag( tag ) + WriteNode( Engine * engine, std::string tag ) + : Node( NodeDef( 1, 0 ), engine ) + , m_tag( tag ) { } void execute() { - //std::cerr << engine() -> time() << ": " << m_tag << " " << input( InputId( 0 ) ) -> lastValueTyped() << std::endl; - printf( "%s: %s %f\n", engine() -> time().asString().c_str(), m_tag.c_str(), input( InputId( 0 ) ) -> lastValueTyped() ); + // std::cerr << engine() -> time() << ": " << m_tag << " " << input( InputId( 0 ) ) -> lastValueTyped() + // << std::endl; + printf( "%s: %s %f\n", engine()->time().asString().c_str(), m_tag.c_str(), + input( InputId( 0 ) )->lastValueTyped() ); } std::string m_tag; @@ -48,13 +51,12 @@ class WriteNode : public Node class DummyPullAdapter : public PullInputAdapter { public: - DummyPullAdapter( Engine * engine, DateTime startTime, - TimeDelta interval, - int tickCount ) : PullInputAdapter( engine ), - m_startTime( startTime ), - m_interval( interval ), - m_tickCount( tickCount ), - m_curTime( DateTime::NONE() ) + DummyPullAdapter( Engine * engine, DateTime startTime, TimeDelta interval, int tickCount ) + : PullInputAdapter( engine ) + , m_startTime( startTime ) + , m_interval( interval ) + , m_tickCount( tickCount ) + , m_curTime( DateTime::NONE() ) { initBuffer(); } @@ -63,7 +65,7 @@ class DummyPullAdapter : public PullInputAdapter { if( count() == m_tickCount ) return false; - + if( m_curTime == DateTime::NONE() ) m_curTime = m_startTime; else @@ -85,16 +87,16 @@ class DummyPullAdapter : public PullInputAdapter else m_curTime += m_interval; - t = m_curTime; + t = m_curTime; value = count(); return true; } - + private: - DateTime m_startTime; + DateTime m_startTime; TimeDelta m_interval; - int m_tickCount; - DateTime m_curTime; + int m_tickCount; + DateTime m_curTime; }; TEST( EngineTest, test_engine_initial ) @@ -105,24 +107,24 @@ TEST( EngineTest, test_engine_initial ) engine.registerInputAdapter( a1 ); engine.registerInputAdapter( a2 ); - + AddNode * addNode = new AddNode( &engine ); - a1 -> linkTo( addNode, InputId( 0 ) ); - a2 -> linkTo( addNode, InputId( 1 ) ); + a1->linkTo( addNode, InputId( 0 ) ); + a2->linkTo( addNode, InputId( 1 ) ); WriteNode * writeNode = new WriteNode( &engine, "TEST" ); - addNode -> linkTo( InputId( 0 ), writeNode, InputId( 0 ) ); + addNode->linkTo( InputId( 0 ), writeNode, InputId( 0 ) ); WriteNode * writeNode2 = new WriteNode( &engine, "A1" ); - a1 -> linkTo( writeNode2, InputId( 0 ) ); + a1->linkTo( writeNode2, InputId( 0 ) ); WriteNode * writeNode3 = new WriteNode( &engine, "A2" ); - a2 -> linkTo( writeNode3, InputId( 0 ) ); + a2->linkTo( writeNode3, InputId( 0 ) ); - addNode -> setRank( 0 ); - writeNode -> setRank( 1 ); - writeNode2 -> setRank( 0 ); - writeNode3 -> setRank( 0 ); + addNode->setRank( 0 ); + writeNode->setRank( 1 ); + writeNode2->setRank( 0 ); + writeNode3->setRank( 0 ); engine.run(); } @@ -131,7 +133,7 @@ TEST( EngineTest, test_engine_initial ) struct WiringNode { - + }; TEST( EngineTest, test_engine_wiring ) diff --git a/cpp/tests/engine/test_partial_switch_csp_type.cpp b/cpp/tests/engine/test_partial_switch_csp_type.cpp index 5a516ae0..7cbccbdf 100644 --- a/cpp/tests/engine/test_partial_switch_csp_type.cpp +++ b/cpp/tests/engine/test_partial_switch_csp_type.cpp @@ -1,252 +1,203 @@ #include #include +#include #include #include -#include -template< typename T, typename B > +template auto getConverter( std::shared_ptr curType ) { - return csp::ConstructibleTypeSwitch::invoke( curType . get(), []( auto tag ) - { - return std::function( []( T val ) { return B(( typename decltype(tag)::type ) ( val )); } ); - } ); + return csp::ConstructibleTypeSwitch::invoke( + curType.get(), []( auto tag ) + { return std::function( []( T val ) { return B( ( typename decltype( tag )::type )( val ) ); } ); } ); } -template< csp::CspType::TypeTraits::_enum v, typename B=long long, typename T=typename csp::CspType::Type::toCType::type > +template::type> void runTypeConversionSupportTest( std::set supportedTypes, std::set unsupportedTypes ) { - ASSERT_EQ( supportedTypes . size() + unsupportedTypes . size(), 10 ); - for( auto t:supportedTypes ) + ASSERT_EQ( supportedTypes.size() + unsupportedTypes.size(), 10 ); + for( auto t : supportedTypes ) { try { - auto converter = getConverter( std::make_shared( t )); + auto converter = getConverter( std::make_shared( t ) ); if( t != csp::CspType::Type::BOOL ) { const bool specialIntCase = ( v == csp::CspType::Type::UINT64 ) && ( t == csp::CspType::Type::INT64 ); if( !specialIntCase ) { - ASSERT_EQ( B( std::numeric_limits::max()), converter( std::numeric_limits::max())); + ASSERT_EQ( B( std::numeric_limits::max() ), converter( std::numeric_limits::max() ) ); } - ASSERT_EQ( std::numeric_limits::min(), converter( std::numeric_limits::min())); + ASSERT_EQ( std::numeric_limits::min(), converter( std::numeric_limits::min() ) ); } else { - ASSERT_EQ( std::numeric_limits::max() != 0, converter( std::numeric_limits::max())); - ASSERT_EQ( std::numeric_limits::min() != 0, converter( std::numeric_limits::min())); + ASSERT_EQ( std::numeric_limits::max() != 0, converter( std::numeric_limits::max() ) ); + ASSERT_EQ( std::numeric_limits::min() != 0, converter( std::numeric_limits::min() ) ); } } - catch( const csp::UnsupportedSwitchType &e ) + catch( const csp::UnsupportedSwitchType & e ) { throw; } } - for( auto t:unsupportedTypes ) + for( auto t : unsupportedTypes ) { - ASSERT_THROW(( getConverter( std::make_shared( t ))), csp::UnsupportedSwitchType ); + ASSERT_THROW( ( getConverter( std::make_shared( t ) ) ), csp::UnsupportedSwitchType ); } } - TEST( ConstructibleTypeSelectorTest, test_basic_type_support ) { - runTypeConversionSupportTest( { - csp::CspType::Type::BOOL, - csp::CspType::Type::INT8, - csp::CspType::Type::UINT8, - csp::CspType::Type::INT16, - csp::CspType::Type::UINT16, - csp::CspType::Type::INT32, - csp::CspType::Type::UINT32, - csp::CspType::Type::INT64, - csp::CspType::Type::UINT64, - csp::CspType::Type::DOUBLE, - }, - {} ); - - runTypeConversionSupportTest( { - csp::CspType::Type::BOOL, - csp::CspType::Type::INT8, - csp::CspType::Type::INT16, - csp::CspType::Type::INT32, - csp::CspType::Type::INT64, - csp::CspType::Type::DOUBLE }, - { csp::CspType::Type::UINT8, - csp::CspType::Type::UINT16, - csp::CspType::Type::UINT32, - csp::CspType::Type::UINT64, + runTypeConversionSupportTest( + { + csp::CspType::Type::BOOL, + csp::CspType::Type::INT8, + csp::CspType::Type::UINT8, + csp::CspType::Type::INT16, + csp::CspType::Type::UINT16, + csp::CspType::Type::INT32, + csp::CspType::Type::UINT32, + csp::CspType::Type::INT64, + csp::CspType::Type::UINT64, + csp::CspType::Type::DOUBLE, + }, + {} ); + + runTypeConversionSupportTest( { csp::CspType::Type::BOOL, csp::CspType::Type::INT8, + csp::CspType::Type::INT16, csp::CspType::Type::INT32, + csp::CspType::Type::INT64, csp::CspType::Type::DOUBLE }, + { + csp::CspType::Type::UINT8, + csp::CspType::Type::UINT16, + csp::CspType::Type::UINT32, + csp::CspType::Type::UINT64, } ); - runTypeConversionSupportTest( { - csp::CspType::Type::BOOL, - csp::CspType::Type::UINT8, - csp::CspType::Type::INT16, - csp::CspType::Type::UINT16, - csp::CspType::Type::INT32, - csp::CspType::Type::UINT32, - csp::CspType::Type::INT64, - csp::CspType::Type::UINT64, - csp::CspType::Type::DOUBLE }, - { - csp::CspType::Type::INT8 - } ); - - runTypeConversionSupportTest( { - csp::CspType::Type::BOOL, - csp::CspType::Type::INT16, - csp::CspType::Type::INT32, - csp::CspType::Type::INT64, - csp::CspType::Type::DOUBLE }, - { csp::CspType::Type::UINT8, - csp::CspType::Type::INT8, - csp::CspType::Type::UINT16, - csp::CspType::Type::UINT32, - csp::CspType::Type::UINT64 - } ); - - runTypeConversionSupportTest( { - csp::CspType::Type::BOOL, - csp::CspType::Type::UINT16, - csp::CspType::Type::INT32, - csp::CspType::Type::UINT32, - csp::CspType::Type::INT64, - csp::CspType::Type::UINT64, - csp::CspType::Type::DOUBLE }, + runTypeConversionSupportTest( + { csp::CspType::Type::BOOL, csp::CspType::Type::UINT8, csp::CspType::Type::INT16, csp::CspType::Type::UINT16, + csp::CspType::Type::INT32, csp::CspType::Type::UINT32, csp::CspType::Type::INT64, csp::CspType::Type::UINT64, + csp::CspType::Type::DOUBLE }, + { csp::CspType::Type::INT8 } ); + + runTypeConversionSupportTest( + { csp::CspType::Type::BOOL, csp::CspType::Type::INT16, csp::CspType::Type::INT32, csp::CspType::Type::INT64, + csp::CspType::Type::DOUBLE }, + { csp::CspType::Type::UINT8, csp::CspType::Type::INT8, csp::CspType::Type::UINT16, csp::CspType::Type::UINT32, + csp::CspType::Type::UINT64 } ); + + runTypeConversionSupportTest( + { csp::CspType::Type::BOOL, csp::CspType::Type::UINT16, csp::CspType::Type::INT32, csp::CspType::Type::UINT32, + csp::CspType::Type::INT64, csp::CspType::Type::UINT64, csp::CspType::Type::DOUBLE }, + { csp::CspType::Type::INT8, csp::CspType::Type::UINT8, csp::CspType::Type::INT16 } ); + + runTypeConversionSupportTest( + { csp::CspType::Type::BOOL, csp::CspType::Type::INT32, csp::CspType::Type::INT64, csp::CspType::Type::DOUBLE }, + { csp::CspType::Type::UINT8, csp::CspType::Type::INT8, csp::CspType::Type::INT16, csp::CspType::Type::UINT16, + csp::CspType::Type::UINT32, csp::CspType::Type::UINT64 } ); + + runTypeConversionSupportTest( { csp::CspType::Type::BOOL, csp::CspType::Type::UINT32, + csp::CspType::Type::INT64, csp::CspType::Type::UINT64, + csp::CspType::Type::DOUBLE }, { - csp::CspType::Type::INT8, - csp::CspType::Type::UINT8, - csp::CspType::Type::INT16 + csp::CspType::Type::INT8, + csp::CspType::Type::UINT8, + csp::CspType::Type::INT16, + csp::CspType::Type::UINT16, + csp::CspType::Type::INT32, } ); - runTypeConversionSupportTest( { - csp::CspType::Type::BOOL, - csp::CspType::Type::INT32, - csp::CspType::Type::INT64, - csp::CspType::Type::DOUBLE }, - { csp::CspType::Type::UINT8, - csp::CspType::Type::INT8, - csp::CspType::Type::INT16, - csp::CspType::Type::UINT16, - csp::CspType::Type::UINT32, - csp::CspType::Type::UINT64 - } ); - - runTypeConversionSupportTest( { - csp::CspType::Type::BOOL, - csp::CspType::Type::UINT32, - csp::CspType::Type::INT64, - csp::CspType::Type::UINT64, - csp::CspType::Type::DOUBLE }, - { - csp::CspType::Type::INT8, - csp::CspType::Type::UINT8, - csp::CspType::Type::INT16, - csp::CspType::Type::UINT16, - csp::CspType::Type::INT32, - } ); + runTypeConversionSupportTest( + { csp::CspType::Type::BOOL, csp::CspType::Type::INT64, csp::CspType::Type::DOUBLE }, + { csp::CspType::Type::UINT8, csp::CspType::Type::INT8, csp::CspType::Type::INT16, csp::CspType::Type::UINT16, + csp::CspType::Type::INT32, csp::CspType::Type::UINT32, csp::CspType::Type::UINT64 } ); - runTypeConversionSupportTest( { - csp::CspType::Type::BOOL, - csp::CspType::Type::INT64, - csp::CspType::Type::DOUBLE }, - { csp::CspType::Type::UINT8, - csp::CspType::Type::INT8, - csp::CspType::Type::INT16, - csp::CspType::Type::UINT16, - csp::CspType::Type::INT32, - csp::CspType::Type::UINT32, - csp::CspType::Type::UINT64 - } ); - - runTypeConversionSupportTest( { - csp::CspType::Type::BOOL, - csp::CspType::Type::UINT64, - csp::CspType::Type::INT64, - csp::CspType::Type::DOUBLE }, - { - csp::CspType::Type::INT8, - csp::CspType::Type::UINT8, - csp::CspType::Type::INT16, - csp::CspType::Type::UINT16, - csp::CspType::Type::INT32, - csp::CspType::Type::UINT32, - - } ); + runTypeConversionSupportTest( + { csp::CspType::Type::BOOL, csp::CspType::Type::UINT64, csp::CspType::Type::INT64, csp::CspType::Type::DOUBLE }, + { + csp::CspType::Type::INT8, + csp::CspType::Type::UINT8, + csp::CspType::Type::INT16, + csp::CspType::Type::UINT16, + csp::CspType::Type::INT32, + csp::CspType::Type::UINT32, + + } ); std::uint64_t aux = std::numeric_limits::max(); - ASSERT_THROW( csp::cast( std::numeric_limits::max()), csp::RangeError ); - ASSERT_THROW( csp::cast( std::numeric_limits::max()), csp::RangeError ); + ASSERT_THROW( csp::cast( std::numeric_limits::max() ), csp::RangeError ); + ASSERT_THROW( csp::cast( std::numeric_limits::max() ), csp::RangeError ); ASSERT_THROW( csp::cast( aux ), csp::RangeError ); ASSERT_THROW( csp::cast( aux ), csp::RangeError ); - runTypeConversionSupportTest( { - csp::CspType::Type::BOOL, - csp::CspType::Type::DOUBLE }, - { - csp::CspType::Type::INT8, - csp::CspType::Type::UINT8, - csp::CspType::Type::INT16, - csp::CspType::Type::UINT16, - csp::CspType::Type::INT32, - csp::CspType::Type::UINT32, - csp::CspType::Type::INT64, - csp::CspType::Type::UINT64, - } ); + runTypeConversionSupportTest( + { csp::CspType::Type::BOOL, csp::CspType::Type::DOUBLE }, + { + csp::CspType::Type::INT8, + csp::CspType::Type::UINT8, + csp::CspType::Type::INT16, + csp::CspType::Type::UINT16, + csp::CspType::Type::INT32, + csp::CspType::Type::UINT32, + csp::CspType::Type::INT64, + csp::CspType::Type::UINT64, + } ); } - -template< typename T1, typename T2 > +template void verifySameTypes() { if( !std::is_same_v ) { static_assert( !std::is_same_v> ); static_assert( !std::is_same_v> ); - CSP_THROW( csp::RuntimeException, "Types mismatch" << typeid( T1 ).name() << "," << typeid( T2 ).name()); + CSP_THROW( csp::RuntimeException, "Types mismatch" << typeid( T1 ).name() << "," << typeid( T2 ).name() ); } } TEST( ArraySwitchTest, test_basic_switch ) { auto uint64Type = csp::CspType::UINT64(); - auto uint64ArrayType = csp::CspArrayType::create( csp::CspType::UINT64()); - auto boolArrayType = csp::CspArrayType::create( csp::CspType::BOOL()); - csp::PrimitiveCspTypeSwitch::invoke( uint64Type.get(), []( auto tag ) - { - if( !std::is_same_v ) - { - CSP_THROW( csp::RuntimeException, "Dummy error" ); - } - } ); + auto uint64ArrayType = csp::CspArrayType::create( csp::CspType::UINT64() ); + auto boolArrayType = csp::CspArrayType::create( csp::CspType::BOOL() ); + csp::PrimitiveCspTypeSwitch::invoke( uint64Type.get(), + []( auto tag ) + { + if( !std::is_same_v ) + { + CSP_THROW( csp::RuntimeException, "Dummy error" ); + } + } ); // Primitive shouldn't support arrays - ASSERT_THROW( csp::PrimitiveCspTypeSwitch::invoke( uint64ArrayType.get(), []( auto tag ) {} ), csp::UnsupportedSwitchType ); + ASSERT_THROW( csp::PrimitiveCspTypeSwitch::invoke( uint64ArrayType.get(), []( auto tag ) {} ), + csp::UnsupportedSwitchType ); // Should work fine - csp::PartialSwitchCspType::invoke( uint64ArrayType.get(), []( auto tag ) - { - verifySameTypes>(); - } ); - csp::PartialSwitchCspType::invoke( uint64ArrayType.get(), []( auto tag ) - { - verifySameTypes>(); - } ); - csp::PartialSwitchCspType::invoke( boolArrayType.get(), []( auto tag ) - { - verifySameTypes>(); - } ); + csp::PartialSwitchCspType::invoke( + uint64ArrayType.get(), + []( auto tag ) { verifySameTypes>(); } ); + csp::PartialSwitchCspType::invoke( + uint64ArrayType.get(), + []( auto tag ) { verifySameTypes>(); } ); + csp::PartialSwitchCspType::invoke( + boolArrayType.get(), + []( auto tag ) { verifySameTypes>(); } ); csp::PartialSwitchCspType::invoke>( - boolArrayType.get(), []( auto tag ) { verifySameTypes>(); } ); + boolArrayType.get(), + []( auto tag ) { verifySameTypes>(); } ); csp::PartialSwitchCspType::invoke>( - uint64ArrayType.get(), []( auto tag ) { verifySameTypes>(); } ); + uint64ArrayType.get(), + []( auto tag ) { verifySameTypes>(); } ); // Should raise since we support only array of bool - ASSERT_THROW( csp::PartialSwitchCspType::invoke>( - uint64ArrayType.get(), []( auto tag ) {} ), csp::UnsupportedSwitchType ); + ASSERT_THROW( csp::PartialSwitchCspType::invoke< + csp::PartialSwitchCspType>( uint64ArrayType.get(), []( auto tag ) {} ), + csp::UnsupportedSwitchType ); // Should raise since we support only array of uint64_t - ASSERT_THROW( csp::PartialSwitchCspType::invoke>( - boolArrayType.get(), []( auto tag ) {} ), csp::UnsupportedSwitchType ); + ASSERT_THROW( csp::PartialSwitchCspType::invoke< + csp::PartialSwitchCspType>( boolArrayType.get(), []( auto tag ) {} ), + csp::UnsupportedSwitchType ); } - diff --git a/cpp/tests/engine/test_tick_buffer.cpp b/cpp/tests/engine/test_tick_buffer.cpp index 443639f3..03e3af5f 100644 --- a/cpp/tests/engine/test_tick_buffer.cpp +++ b/cpp/tests/engine/test_tick_buffer.cpp @@ -28,7 +28,7 @@ TEST( TickBufferTest, test_resize ) buffer.push_back( 1 ); buffer.push_back( 2 ); - //resize before full + // resize before full buffer.growBuffer( 5 ); buffer.push_back( 3 ); @@ -37,32 +37,32 @@ TEST( TickBufferTest, test_resize ) ASSERT_EQ( buffer.valueAtIndex( 0 ), 3 ); ASSERT_EQ( buffer.valueAtIndex( 1 ), 2 ); ASSERT_EQ( buffer.valueAtIndex( 2 ), 1 ); - ASSERT_THROW( buffer.valueAtIndex( 3 ), csp::RangeError ); //exception + ASSERT_THROW( buffer.valueAtIndex( 3 ), csp::RangeError ); // exception buffer.push_back( 4 ); buffer.push_back( 5 ); buffer.push_back( 6 ); for( int i = 0; i < 5; ++i ) - ASSERT_EQ( buffer[ i ], 6 - i ) << i; + ASSERT_EQ( buffer[i], 6 - i ) << i; - //resize after full + // resize after full buffer.growBuffer( 8 ); ASSERT_EQ( buffer.numTicks(), 5 ); ASSERT_EQ( buffer.capacity(), 8 ); for( int i = 0; i < 5; ++i ) - ASSERT_EQ( buffer[ i ], 6 - i ); - ASSERT_THROW( buffer.valueAtIndex( 5 ), csp::RangeError ); //exception + ASSERT_EQ( buffer[i], 6 - i ); + ASSERT_THROW( buffer.valueAtIndex( 5 ), csp::RangeError ); // exception buffer.push_back( 7 ); ASSERT_EQ( buffer.valueAtIndex( 0 ), 7 ); - //TODO: THIS TEST IS BROKEN, fix it + // TODO: THIS TEST IS BROKEN, fix it ASSERT_EQ( buffer.valueAtIndex( 5 ), 2 ); } TEST( TickBufferTest, test_flatten ) { - int * values_wrap, * values_nowrap, * values_single; + int *values_wrap, *values_nowrap, *values_single; { csp::TickBuffer buffer; buffer.growBuffer( 5 ); @@ -84,18 +84,18 @@ TEST( TickBufferTest, test_flatten ) buffer.push_back( 7 ); for( int i = 0; i < 5; ++i ) - ASSERT_EQ( buffer[ i ], 7 - i ); + ASSERT_EQ( buffer[i], 7 - i ); values_wrap = buffer.flatten( 4, 0 ); ASSERT_THROW( buffer.flatten( 5, 0 ), csp::RangeError ); } - ASSERT_EQ( values_single[ 0 ], 0 ); + ASSERT_EQ( values_single[0], 0 ); for( int i = 0; i < 5; ++i ) { - ASSERT_EQ( values_wrap[ i ], i + 3 ); - ASSERT_EQ( values_nowrap[ i ], i ); + ASSERT_EQ( values_wrap[i], i + 3 ); + ASSERT_EQ( values_nowrap[i], i ); } } diff --git a/cpp/tests/engine/test_time_series.cpp b/cpp/tests/engine/test_time_series.cpp index 44dcb7fb..44c09f80 100644 --- a/cpp/tests/engine/test_time_series.cpp +++ b/cpp/tests/engine/test_time_series.cpp @@ -1,29 +1,32 @@ -#include #include +#include using namespace csp; struct Foo { - Foo( int a_, int b_ ) : a( a_ ), b( b_ ) - {} - + Foo( int a_, int b_ ) + : a( a_ ) + , b( b_ ) + { + } + int a; int b; }; TEST( TimeSeriesTest, test_basic_functionality ) { - DateTime time( 2017, 1, 1 ); + DateTime time( 2017, 1, 1 ); TimeDelta delta = TimeDelta::fromSeconds( 1 ); - + for( int tickCountSize = 1; tickCountSize < 10; tickCountSize += 2 ) { TimeSeriesTyped ts_d; TimeSeries & ts_d_ref = ts_d; ts_d_ref.setTickCountPolicy( tickCountSize ); - - for( int i =0; i < tickCountSize; ++i ) + + for( int i = 0; i < tickCountSize; ++i ) { ts_d_ref.addTickTyped( time + delta * i, i ); EXPECT_EQ( ts_d_ref.lastValueTyped(), i ); @@ -31,18 +34,18 @@ TEST( TimeSeriesTest, test_basic_functionality ) EXPECT_EQ( ts_d_ref.count(), i + 1 ); EXPECT_EQ( ts_d_ref.numTicks(), i + 1 ); - for( int j = 0; j<=i; ++j ) + for( int j = 0; j <= i; ++j ) { EXPECT_EQ( ts_d_ref.valueAtIndex( j ), i - j ); - EXPECT_EQ( ts_d_ref.timeAtIndex( j ), time + delta*(i - j )); + EXPECT_EQ( ts_d_ref.timeAtIndex( j ), time + delta * ( i - j ) ); } } EXPECT_THROW( ts_d_ref.valueAtIndex( tickCountSize ), std::exception ); } - TimeSeriesTyped ts_b; - TimeSeriesTyped > ts_s; + TimeSeriesTyped ts_b; + TimeSeriesTyped> ts_s; ASSERT_EQ( ts_b.numTicks(), 0 ); ASSERT_EQ( ts_b.count(), 0 ); @@ -59,15 +62,15 @@ TEST( TimeSeriesTest, test_basic_functionality ) for( int i = 0; i < 25; i++ ) { - ts_b.addTick( time + delta * i, true ); + ts_b.addTick( time + delta * i, true ); } - + ASSERT_EQ( ts_b.numTicks(), 10 ); ASSERT_EQ( ts_b.count(), 25 ); for( int i = 0; i < 25; i++ ) { - ts_s.addTick( time + delta * i, std::make_shared( 1, 2 ) ); + ts_s.addTick( time + delta * i, std::make_shared( 1, 2 ) ); } ASSERT_EQ( ts_s.numTicks(), 16 ); // buffer gets resized to nextpow2, so 16 ticks are available diff --git a/cpp/tests/engine/test_window_buffer.cpp b/cpp/tests/engine/test_window_buffer.cpp index c3978094..316e6ac6 100644 --- a/cpp/tests/engine/test_window_buffer.cpp +++ b/cpp/tests/engine/test_window_buffer.cpp @@ -1,7 +1,7 @@ #include #include -//Note clear why windows linker is looknig for this +// Note clear why windows linker is looknig for this #ifdef WIN32 bool csp::DialectGenericType::operator==( struct csp::DialectGenericType const & ) const { return false; } #endif @@ -9,44 +9,49 @@ bool csp::DialectGenericType::operator==( struct csp::DialectGenericType const & TEST( WindowBufferTest, test_time_window ) { csp::VariableSizeWindowBuffer buffer; - for (int i = 1; i <= 10; i++) buffer.push(i); - for (int i = 1; i <= 8; i++) buffer.pop_left(); - for (int i = 11; i <= 18; i++) buffer.push(i); - for (int i = 1; i <= 3; i++) buffer.pop_right(); - + for( int i = 1; i <= 10; i++ ) + buffer.push( i ); + for( int i = 1; i <= 8; i++ ) + buffer.pop_left(); + for( int i = 11; i <= 18; i++ ) + buffer.push( i ); + for( int i = 1; i <= 3; i++ ) + buffer.pop_right(); + ASSERT_EQ( buffer.capacity(), 16 ); // 7 values in buffer but does not downsize - ASSERT_EQ( buffer[0], 15); - ASSERT_EQ( buffer[6], 9); // 6 ticks away + ASSERT_EQ( buffer[0], 15 ); + ASSERT_EQ( buffer[6], 9 ); // 6 ticks away ASSERT_EQ( buffer.count(), 7 ); } TEST( WindowBufferTest, test_tick_window ) { - csp::FixedSizeWindowBuffer buffer(8); + csp::FixedSizeWindowBuffer buffer( 8 ); std::vector rem; - for (int i = 1; i <= 10; i++) + for( int i = 1; i <= 10; i++ ) { - if (buffer.full() && rem.size() < 8) - rem.push_back(buffer.pop_left()); - buffer.push(i); + if( buffer.full() && rem.size() < 8 ) + rem.push_back( buffer.pop_left() ); + buffer.push( i ); } ASSERT_EQ( buffer.count(), 8 ); ASSERT_EQ( rem.size(), 2 ); ASSERT_EQ( rem[0], 1 ); ASSERT_EQ( rem[1], 2 ); - for (int i = 1; i <= 5; i++) buffer.pop_left(); + for( int i = 1; i <= 5; i++ ) + buffer.pop_left(); ASSERT_EQ( buffer.count(), 3 ); - for (int i = 11; i <= 25; i++) + for( int i = 11; i <= 25; i++ ) { - if (buffer.full() && rem.size() < 8) - rem.push_back(buffer.pop_left()); - buffer.push(i); + if( buffer.full() && rem.size() < 8 ) + rem.push_back( buffer.pop_left() ); + buffer.push( i ); } ASSERT_EQ( buffer.capacity(), 8 ); // 8 values in buffer - ASSERT_EQ( buffer[0], 25); - ASSERT_EQ( buffer[7], 18); // 7 ticks away + ASSERT_EQ( buffer[0], 25 ); + ASSERT_EQ( buffer[7], 18 ); // 7 ticks away ASSERT_EQ( buffer.count(), 8 ); // Removals is full, discards unimportant values @@ -54,5 +59,3 @@ TEST( WindowBufferTest, test_tick_window ) ASSERT_EQ( rem[0], 1 ); ASSERT_EQ( rem[7], 13 ); } - -