aoflagger-v3.5.1/0000775000175000017500000000000015146315735011771 5ustar oleoleaoflagger-v3.5.1/CMakeLists.txt0000664000175000017500000005232415132202045014517 0ustar oleolecmake_minimum_required(VERSION 3.12) project(AOFlagger LANGUAGES CXX) include(CTest) include(ExternalProject) include(CMakeVersionInfo.txt) set(AOFLAGGER_VERSION "${AOFLAGGER_VERSION_MAJOR}.${AOFLAGGER_VERSION_MINOR}.${AOFLAGGER_VERSION_SUBMINOR}" ) # Some CMake 'find package' files are stored in this directory set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) find_package(PkgConfig) pkg_check_modules(SIGCXX sigc++-2.0) option(ENABLE_GUI "Build rfigui and aoqplot tools" ON) if(ENABLE_GUI) pkg_check_modules(GTKMM gtkmm-4.0>=4.0.0) if(GTKMM_FOUND) add_definitions(-DHAVE_GTKMM) else() message( FATAL_ERROR "The graphical user interface library GTKMM was not found. To build AOFlagger without GUI, add " "-DENABLE_GUI=OFF to cmake.") endif(GTKMM_FOUND) endif() # Find and include git submodules find_package(Git QUIET) if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") # Update submodules as needed option(GIT_SUBMODULE "Check submodules during build" ON) if(GIT_SUBMODULE) message(STATUS "Submodule update") execute_process( COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive --checkout --depth 1 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULT_VARIABLE GIT_SUBMOD_RESULT) if(NOT GIT_SUBMOD_RESULT EQUAL "0") message( FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules" ) endif() endif() endif() # Python needs to be found before pybind11 to avoid finding different versions # of python find_package( Python COMPONENTS Interpreter Development REQUIRED) message(STATUS "Using python version ${Python_VERSION}") include_directories(SYSTEM ${Python_INCLUDE_DIRS}) list(APPEND ExternalSubmoduleDirectories aocommon) foreach(ExternalSubmodule IN LISTS ExternalSubmoduleDirectories) if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${ExternalSubmodule}) message( FATAL_ERROR "The external submodule '${ExternalSubmodule}' is missing in the external/ subdirectory. " "This is likely the result of downloading a git tarball without submodules. " "This is not supported: git tarballs do not provide the required versioning " "information for the submodules. Please perform a git clone of this repository." ) endif() endforeach() # Include aocommon headers include_directories("${CMAKE_SOURCE_DIR}/external/aocommon/include") # Include pybind11 headers find_package(pybind11 CONFIG REQUIRED) include_directories(SYSTEM ${pybind11_INCLUDE_DIRS}) find_package( HDF5 COMPONENTS CXX REQUIRED) add_definitions(${HDF5_DEFINITIONS}) include_directories(SYSTEM ${HDF5_INCLUDE_DIR}) set(CASACORE_MAKE_REQUIRED_EXTERNALS_OPTIONAL TRUE) find_package(Casacore REQUIRED COMPONENTS casa ms tables measures) include_directories(SYSTEM ${CASACORE_INCLUDE_DIRS}) find_package(CFITSIO REQUIRED) include_directories(SYSTEM ${CFITSIO_INCLUDE_DIR}) find_path( FFTW3_INCLUDE_DIR NAMES fftw3.h HINTS ENV FFTW3_INCLUDE) find_library(GSL_LIB NAMES gsl) find_library(GSL_CBLAS_LIB NAMES gslcblas) find_path(GSL_INCLUDE_DIR NAMES gsl/gsl_version.h) if(GSL_INCLUDE_DIR) include_directories(SYSTEM ${GSL_INCLUDE_DIR}) endif(GSL_INCLUDE_DIR) find_package(Threads REQUIRED) find_package(PNG REQUIRED) # AOFlagger uses the Boost 'system' and 'date_time' components, which have been # header-only since Boost 1.69 and 1.73, respectively. Using "COMPONENTS system # date_time" is thus no longer needed since Boost 1.73. Since Boost 1.89, those # COMPONENTS no longer work and raise errors. find_package(Boost 1.73.0 REQUIRED) include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) find_library(FFTW3_LIB fftw3 REQUIRED HINTS ENV FFTW3_LIB) include_directories(SYSTEM ${FFTW3_INCLUDE_DIR}) enable_language(Fortran OPTIONAL) find_package(BLAS REQUIRED) find_package(LAPACK REQUIRED) find_package(Lua 5.3 REQUIRED) include_directories(SYSTEM ${LUA_INCLUDE_DIR}) # The following stuff will set the "rpath" correctly, so that LD_LIBRARY_PATH # doesn't have to be set. # use, i.e. don't skip the full RPATH for the build tree set(CMAKE_SKIP_BUILD_RPATH FALSE) # when building, don't use the install RPATH already (but later on when # installing) set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") # add the automatically determined parts of the RPATH which point to directories # outside the build tree to the install RPATH set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) # the RPATH to be used when installing, but only if it's not a system directory list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir) if("${isSystemDir}" STREQUAL "-1") set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") endif("${isSystemDir}" STREQUAL "-1") set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED YES) set(CMAKE_CXX_EXTENSIONS NO) add_compile_options( -DNDEBUG -O3 -Wall -Wvla -Wzero-as-null-pointer-constant -Wnon-virtual-dtor -Wduplicated-branches -Wundef -Wvla -Wpointer-arith -Wextra -Wno-unused-parameter) if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") # GCC 8.x requires linking with stdc++fs for the filesystem library # https://gcc.gnu.org/onlinedocs/gcc-9.1.0/libstdc++/manual/manual/status.html#status.iso.2017 if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) link_libraries(stdc++fs) elseif(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0) message( FATAL_ERROR "The GCC version is too old, upgrade to GCC 8.0 or newer") endif() endif() if(CMAKE_BUILD_TYPE STREQUAL "Debug") message(STATUS "Debug build selected: setting linking flag --no-undefined") string(APPEND CMAKE_SHARED_LINKER_FLAGS " -Wl,--no-undefined") endif() include(CheckCXXCompilerFlag) include(CheckSymbolExists) include(CheckCXXSymbolExists) include(CheckIncludeFileCXX) check_symbol_exists(posix_fallocate "fcntl.h" HAVE_POSIX_FALLOCATE) check_cxx_symbol_exists(exp10 "cmath" HAVE_EXP10) if(HAVE_POSIX_FALLOCATE) add_definitions(-DHAVE_POSIX_FALLOCATE) endif(HAVE_POSIX_FALLOCATE) # User may optionally set `TARGET_CPU` if `PORTABLE=OFF` option(PORTABLE "Build portable binaries (with slightly decreased performance)" OFF) include(external/aocommon/CMake/SetTargetCPU.cmake) if(GSL_LIB AND GSL_CBLAS_LIB AND GSL_INCLUDE_DIR) add_definitions(-DHAVE_GSL) link_libraries(${GSL_LIB} ${GSL_CBLAS_LIB}) message(STATUS "GSL found.") endif( GSL_LIB AND GSL_CBLAS_LIB AND GSL_INCLUDE_DIR) configure_file(version.h.in version.h) include_directories(${CMAKE_CURRENT_BINARY_DIR}) set(AOQPLOT_FILES aoqplot/aoqplotwindow.cpp aoqplot/baselineplotpage.cpp aoqplot/datawindow.cpp aoqplot/grayscaleplotpage.cpp aoqplot/histogrampage.cpp aoqplot/timefrequencyplotpage.cpp aoqplot/twodimensionalplotpage.cpp aoqplot/controllers/aoqplotcontroller.cpp aoqplot/controllers/aoqplotpagecontroller.cpp aoqplot/controllers/baselinepagecontroller.cpp aoqplot/controllers/heatmappagecontroller.cpp aoqplot/controllers/histogrampagecontroller.cpp aoqplot/controllers/tfpagecontroller.cpp) set(PLOT_FILES rfigui/maskedheatmap.cpp plot/colorscale.cpp plot/heatmap.cpp plot/horizontalplotscale.cpp plot/legend.cpp plot/plotbase.cpp plot/plotpropertieswindow.cpp plot/plotwidget.cpp plot/statistics.cpp plot/title.cpp plot/vectorimage.cpp plot/verticalplotscale.cpp plot/xyplot.cpp) set(PLOT_TEST_FILES test/plot/statisticstest.cpp) set(GUI_FILES rfigui/controllers/imagecomparisoncontroller.cpp rfigui/controllers/rfiguicontroller.cpp rfigui/gotowindow.cpp rfigui/imagepropertieswindow.cpp rfigui/initialization.cpp rfigui/opendialog.cpp rfigui/plotframe.cpp rfigui/plotwindow.cpp rfigui/progresswindow.cpp rfigui/rangeinputdialog.cpp rfigui/rfiguimenu.cpp rfigui/rfiguiwindow.cpp rfigui/settings.cpp rfigui/simulatedialog.cpp rfigui/strategyeditor.cpp util/rfiplots.cpp util/multiplot.cpp ${PLOT_FILES} ${AOQPLOT_FILES}) set(IMAGING_FILES imaging/uvimager.cpp imaging/model.cpp) set(INTERFACE_FILES interface/aoflagger.cpp interface/flagmask.cpp interface/imageset.cpp interface/qualitystatistics.cpp interface/strategy.cpp) set(LUA_FILES lua/datawrapper.cpp lua/default-strategy.cpp lua/functions.cpp lua/functionswrapper.cpp lua/luastrategy.cpp lua/optionsfunction.cpp lua/scriptdata.cpp lua/telescopefile.cpp) set(MSIO_FILES msio/averagingmsreader.cpp msio/baselinematrixloader.cpp msio/baselinereader.cpp msio/directbaselinereader.cpp msio/fitsfile.cpp msio/memorybaselinereader.cpp msio/pngfile.cpp msio/reorderingbaselinereader.cpp msio/rspreader.cpp msio/singlebaselinefile.cpp msio/spatialtimeloader.cpp) set(STRUCTURES_FILES structures/image2d.cpp structures/mask2d.cpp structures/msiterator.cpp structures/msmetadata.cpp structures/samplerow.cpp structures/stokesimager.cpp structures/timefrequencydata.cpp) set(QUALITY_FILES quality/collector.cpp quality/combine.cpp quality/histogramcollection.cpp quality/histogramtablesformatter.cpp quality/operations.cpp quality/qualitytablesformatter.cpp quality/rayleighfitter.cpp quality/statisticscollection.cpp) set(ALGORITHMS_FILES algorithms/antennaselector.cpp algorithms/baselineselector.cpp algorithms/baselinetimeplaneimager.cpp algorithms/combinatorialthresholder.cpp algorithms/fringetestcreater.cpp algorithms/highpassfilter.cpp algorithms/morphology.cpp algorithms/sinusfitter.cpp algorithms/siroperator.cpp algorithms/morphologicalflagger.cpp algorithms/sumthreshold.cpp algorithms/sumthresholdmissing.cpp algorithms/svdmitigater.cpp algorithms/testsetgenerator.cpp algorithms/thresholdconfig.cpp algorithms/thresholdtools.cpp algorithms/timefrequencystatistics.cpp algorithms/sisco/sisco.cpp algorithms/sisco/predictresidual.cpp) set(PYTHON_FILES python/pythonstrategy.cpp python/pyfunctions.cpp) set(IMAGESETS_FILES imagesets/bhfitsimageset.cpp imagesets/filterbankset.cpp imagesets/fitsimageset.cpp imagesets/h5imageset.cpp imagesets/imageset.cpp imagesets/indexableset.cpp imagesets/msimageset.cpp imagesets/multibandmsimageset.cpp imagesets/parmimageset.cpp imagesets/pngreader.cpp imagesets/rfibaselineset.cpp imagesets/sdhdfimageset.cpp) set(UTIL_FILES plot/colormap.cpp util/logger.cpp util/ffttools.cpp util/integerdomain.cpp util/plot.cpp util/rng.cpp util/stopwatch.cpp) set(ALL_LIBRARIES ${CASACORE_LIBRARIES} ${FFTW3_LIB} ${CFITSIO_LIBRARY} ${Python_LIBRARIES} ${PNG_LIBRARIES} ${LUA_LIBRARIES} ${Boost_LIBRARIES} ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${HDF5_CXX_LIBRARIES} ${HDF5_C_LIBRARIES}) if(GTKMM_FOUND) message(STATUS "GTKMM found.") include_directories(SYSTEM ${GTKMM_INCLUDE_DIRS}) link_directories(${GTKMM_LIBDIR}) set(ALL_LIBRARIES ${ALL_LIBRARIES} ${GTKMM_LIBRARIES}) endif(GTKMM_FOUND) if(SIGCXX_FOUND) message(STATUS "SIGCXX found.") include_directories(SYSTEM ${SIGCXX_INCLUDE_DIRS}) link_directories(${SIGCXX_LIBDIR}) set(ALL_LIBRARIES ${ALL_LIBRARIES} ${SIGCXX_LIBRARIES}) endif(SIGCXX_FOUND) if(GTKMM_FOUND) set(AOFLAGGER_PLOT_FILES ${PLOT_FILES}) set(AOFLAGGER_PLOT_TEST_FILES ${PLOT_TEST_FILES}) endif(GTKMM_FOUND) set(ALL_NON_GUI_FILES ${ALGORITHMS_FILES} ${AOFLAGGER_PLOT_FILES} ${IMAGESETS_FILES} ${IMAGING_FILES} ${INTERFACE_FILES} ${LUA_FILES} ${MSIO_FILES} ${PYTHON_FILES} ${QUALITY_FILES} ${STRUCTURES_FILES} ${UTIL_FILES}) add_library(aoflagger-lib SHARED ${ALL_NON_GUI_FILES}) set_target_properties(aoflagger-lib PROPERTIES SOVERSION 0) set_target_properties(aoflagger-lib PROPERTIES OUTPUT_NAME aoflagger) if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY) # By default, CMAKE_LIBRARY_OUTPUT_DIRECTORY is not defined. -> Store # libaoflagger.so in lib/ and add the strategies in lib/aoflagger/strategies, # so libaoflagger can find them. Using the current directory is not possible, # since it already contains the 'aoflagger' binary, which has the same name as # the 'aoflagger' directory. file(MAKE_DIRECTORY lib/aoflagger) file(COPY data/strategies DESTINATION lib/aoflagger) set_target_properties(aoflagger-lib PROPERTIES LIBRARY_OUTPUT_DIRECTORY lib) else() # When CMAKE_LIBRARY_OUTPUT_DIRECTORY is set, ensure that libaoflagger.so can # find the strategy files by adding them to that directory. (When building a # python binary wheel, this copy is not necessary even though setup.py uses # CMAKE_LIBRARY_OUTPUT_DIRECTORY: setup.py copies the original strategy files # into the wheel.) file(MAKE_DIRECTORY ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/aoflagger) file(COPY data/strategies DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/aoflagger) endif() target_link_libraries(aoflagger-lib ${ALL_LIBRARIES}) # When building a docker image for an application that uses AOFlagger, like DP3, # the 'core_library' component allows installing only the core library using: # cmake --install --component core_library install( TARGETS aoflagger-lib DESTINATION lib COMPONENT core_library) add_executable(aoquality applications/aoquality.cpp) target_link_libraries(aoquality aoflagger-lib ${ALL_LIBRARIES}) install(TARGETS aoquality DESTINATION bin) if(GTKMM_FOUND) add_library(aoflaggergui OBJECT ${GUI_FILES}) set(AOFLAGGERGUI_OBJECT $) add_executable(rfigui applications/rfigui.cpp ${AOFLAGGERGUI_OBJECT}) target_link_libraries(rfigui aoflagger-lib ${ALL_LIBRARIES}) install(TARGETS rfigui DESTINATION bin) add_executable(aoqplot applications/aoqplot.cpp ${AOFLAGGERGUI_OBJECT}) target_link_libraries(aoqplot aoflagger-lib ${ALL_LIBRARIES}) install(TARGETS aoqplot DESTINATION bin) add_executable(badstations applications/badstations.cpp) target_link_libraries(badstations aoflagger-lib ${ALL_LIBRARIES}) endif(GTKMM_FOUND) add_executable( aoflagger-bin applications/aoflagger.cpp aoluarunner/options.cpp aoluarunner/runner.cpp aoluarunner/baselineiterator.cpp aoluarunner/writethread.cpp) set_target_properties(aoflagger-bin PROPERTIES OUTPUT_NAME aoflagger) target_link_libraries(aoflagger-bin aoflagger-lib ${ALL_LIBRARIES}) install(TARGETS aoflagger-bin DESTINATION bin) # A number of files perform the 'core' high-performance floating point # operations. In these files, NaNs are avoided and thus -ffast-math is allowed. # Note that visibilities can be NaN hence this can not be turned on for all # files. set_source_files_properties( algorithms/sumthreshold.cpp algorithms/sumthresholdmissing.cpp PROPERTIES COMPILE_FLAGS "-ffast-math -funroll-loops") install( FILES interface/aoflagger.h DESTINATION include COMPONENT core_library) install(DIRECTORY data/icons data/applications DESTINATION share) install( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/data/strategies DESTINATION share/aoflagger COMPONENT core_library FILES_MATCHING PATTERN *.lua) # Set-up cmake configuration files configure_file("${PROJECT_SOURCE_DIR}/cmake/config/aoflagger-config.cmake.in" "${PROJECT_BINARY_DIR}/CMakeFiles/aoflagger-config.cmake" @ONLY) configure_file( "${PROJECT_SOURCE_DIR}/cmake/config/aoflagger-config-version.cmake.in" "${PROJECT_BINARY_DIR}/CMakeFiles/aoflagger-config-version.cmake" @ONLY) install( FILES "${PROJECT_BINARY_DIR}/CMakeFiles/aoflagger-config.cmake" "${PROJECT_BINARY_DIR}/CMakeFiles/aoflagger-config-version.cmake" DESTINATION share/aoflagger/cmake COMPONENT core_library) add_subdirectory(python) add_subdirectory(cpack) add_custom_target( sphinxdoc make html WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/doc COMMENT "Generating documentation with Sphinx" VERBATIM) find_package(Doxygen QUIET) if(DOXYGEN_FOUND) # add target to generate API documentation with Doxygen configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) message(STATUS "Doxygen found: API documentation can be compiled.") add_custom_target( doxygendoc ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating API documentation with Doxygen" VERBATIM) add_custom_target(doc DEPENDS doxygendoc sphinxdoc) else() message(STATUS "Doxygen not found: API documentation can not compiled.") add_custom_target(doc DEPENDS sphinxdoc) endif() # This is just for development: it is to convert the default Lua stategy into a # c++ array. add_custom_target( strategy xxd -i data/strategies/generic-default.lua > ${CMAKE_CURRENT_SOURCE_DIR}/lua/default-strategy.cpp WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Converting default strategy to C code" VERBATIM) # TODO(RAP-368) Make a proper integration test target. option(ENABLE_TESTS "Enable the integration tests. This requires downloading test data." OFF) if(ENABLE_TESTS) find_package(Boost 1.56.0 REQUIRED COMPONENTS unit_test_framework) add_executable( runtests test/runtests.cpp test/experiments/defaultstrategyspeedtest.cpp test/experiments/highpassfilterexperiment.cpp test/experiments/tthroughput.cpp test/lua/defaultstrategytest.cpp test/lua/flagnanstest.cpp test/lua/tmetadata.cpp test/lua/tscript.cpp test/lua/optionsfunctiontest.cpp test/lua/telescopefiletest.cpp test/interface/interfacetest.cpp ${AOFLAGGER_PLOT_TEST_FILES} test/quality/qualitytablesformattertest.cpp test/quality/statisticscollectiontest.cpp test/quality/statisticsderivatortest.cpp test/algorithms/convolutionstest.cpp test/algorithms/dilationtest.cpp test/algorithms/highpassfiltertest.cpp test/algorithms/medianwindow.cpp test/algorithms/noisestatisticstest.cpp test/algorithms/siroperatortest.cpp test/algorithms/sumthresholdmissingtest.cpp test/algorithms/sumthresholdtest.cpp test/algorithms/testtools.cpp test/algorithms/thresholdtoolstest.cpp test/algorithms/tthresholdconfig.cpp test/msio/tbaselinereader.cpp test/structures/timage2d.cpp test/structures/tantennainfo.cpp test/structures/tearthposition.cpp test/structures/tfieldinfo.cpp test/structures/ttimefrequencydata.cpp test/structures/ttimefrequencydataoperations.cpp test/structures/tmask2d.cpp test/structures/tversionstring.cpp test/util/numberparsertest.cpp) target_link_libraries(runtests aoflagger-lib ${ALL_LIBRARIES} Boost::unit_test_framework) target_include_directories(runtests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) add_dependencies(runtests concatenate_frequency_integration_test_ms) add_test( runtests runtests -f JUNIT -k runtests.xml --catch_system_error=yes) add_custom_target( check COMMAND runtests DEPENDS runtests) set(TEST_DATA_DIR "${CMAKE_BINARY_DIR}/test_data") configure_file(${CMAKE_SOURCE_DIR}/test/config.h.in ${CMAKE_BINARY_DIR}/test/config.h) # Test data for both unit and integration tests. ExternalProject_Add( concatenate_frequency_integration_test_ms EXCLUDE_FROM_ALL URL https://support.astron.nl/software/ci_data/aoflagger/L228163_SB150_SB151_uv.dppp.MS.tar.gz URL_HASH SHA256=1693ae6a56e22b41439246bca30edd1d32119391ac932ade45d0b748ba476e99 SOURCE_DIR "${TEST_DATA_DIR}/concatenate_frequency" CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "") # Set-up cmake configuration for pytest. set(INTEGRATION_DIR test/integration) set(BINARY_INTEGRATION_DIR ${CMAKE_BINARY_DIR}/${INTEGRATION_DIR}) file(MAKE_DIRECTORY ${BINARY_INTEGRATION_DIR}) configure_file(${CMAKE_SOURCE_DIR}/scripts/test/testconfig.py.in ${BINARY_INTEGRATION_DIR}/testconfig.py) configure_file(${CMAKE_SOURCE_DIR}/scripts/test/utils.py.in ${BINARY_INTEGRATION_DIR}/utils.py) ExternalProject_Add( base_integration_test_ms URL https://support.astron.nl/software/ci_data/aoflagger/3C196_spw5_sub1-WSRT.MS.tar.gz URL_HASH SHA256=e516c936058ba19166f5f192d2c68e83df9ca74987422eea9c158b0905ec2f85 SOURCE_DIR "${TEST_DATA_DIR}/base/3C196_spw5_sub1-WSRT.MS" CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "") execute_process( COMMAND python3 -m pytest WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} OUTPUT_VARIABLE PYTEST_OUTPUT) if(NOT ${PYTEST_OUTPUT} MATCHES "no tests ran") message( FATAL_ERROR "Could not run pytest.\n" "Please install pytest or disable testing using -DENABLE_TESTS=Off") endif() function(add_pytest) foreach(TEST ${ARGN}) set(NAME "integration/${TEST}") add_test( NAME "${NAME}" COMMAND python3 -m pytest -v --junitxml=${CMAKE_BINARY_DIR}/${TEST}.xml "${CMAKE_SOURCE_DIR}/${INTEGRATION_DIR}/${TEST}.py" WORKING_DIRECTORY "${BINARY_INTEGRATION_DIR}") set_tests_properties( "${NAME}" PROPERTIES DEPENDS concatenate_frequency_integration_test_ms DEPENDS aoflagger ENVIRONMENT PYTHONPATH=${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}) endforeach() endfunction() add_pytest(tBase tConcatenateFrequency tPythonInterface) endif() aoflagger-v3.5.1/applications/0000775000175000017500000000000015146315735014457 5ustar oleoleaoflagger-v3.5.1/applications/rfigui.cpp0000664000175000017500000001476715106657110016457 0ustar oleole#include "../rfigui/rfiguiwindow.h" #include "../rfigui/initialization.h" #include "../rfigui/controllers/imagecomparisoncontroller.h" #include "../rfigui/controllers/rfiguicontroller.h" #include "../util/logger.h" #include "../util/progress/stdoutreporter.h" #include "../imagesets/msimageset.h" #include "../imagesets/msoptions.h" #include #include #include #include #include #include #include #include struct SavedBaseline { size_t a1Index, a2Index, bandIndex, sequenceIndex; std::string filename; bool operator<(const SavedBaseline& rhs) const { return std::tie(a1Index, a2Index, bandIndex, sequenceIndex, filename) < std::tie(rhs.a1Index, rhs.a2Index, rhs.bandIndex, rhs.sequenceIndex, rhs.filename); } }; static void run(int argc, char* argv[]) { int argi = 1; std::vector filenames; std::set savedBaselines; bool interactive = true; std::string dataColumnName = "DATA"; bool plotFlags = true; std::optional intervalStart, intervalEnd; while (argi < argc) { if (argv[argi][0] == '-') { std::string p; if (argv[argi][1] == '-') p = &argv[argi][2]; else p = &argv[argi][1]; if (p == "h" || p == "help" || p == "?") { Logger::Info << "The RFIGui is a program to analyze the time-frequency " "information in radio astronomical observations.\n" << "It is written by André Offringa (offringa@gmail.com) and " "published under the GPL 3.\n" << "This program is part of AOFlagger " << AOFLAGGER_VERSION_STR << " (" << AOFLAGGER_VERSION_DATE_STR << ")\n\n" << "Syntax:\n" << " rfigui [-option1 [-option2 ...]] [measurement set]\n\n" << "The main window will be opened when no parameters are " "specified.\n" << "Possible options:\n" << " -help\n" << " Display this help message and exit.\n" << " -version\n" << " Display AOFlagger version and exit.\n" << " -v\n" << " Verbose logging.\n" << " -save-baseline " "\n" << " Save the selected baseline to the given filename. The " "parameter can be specified\n" << " multiple times to save multiple baselines in one run. When " "this parameter is specified,\n" << " the main window will not open and the program will exit " "after saving the requested baselines.\n" << " -data-column \n" << " Open the selected column name.\n" << " -hide-flags / -show-flags\n" << " Do not (or do) plot the flag mask on top of the data. " "Default: plot them.\n"; return; } else if (p == "version") { Logger::Info << "AOFlagger " << AOFLAGGER_VERSION_STR << " (" << AOFLAGGER_VERSION_DATE_STR << ")\n"; return; } else if (p == "save-baseline") { SavedBaseline sb; sb.filename = argv[argi + 1]; sb.a1Index = atoi(argv[argi + 2]); sb.a2Index = atoi(argv[argi + 3]); sb.bandIndex = atoi(argv[argi + 4]); sb.sequenceIndex = atoi(argv[argi + 5]); savedBaselines.insert(sb); interactive = false; argi += 5; } else if (p == "data-column") { ++argi; dataColumnName = argv[argi]; } else if (p == "hide-flags") { plotFlags = false; } else if (p == "show-flags") { plotFlags = true; } else if (p == "v") { Logger::SetVerbosity(Logger::VerboseVerbosity); } else { Logger::Error << "Unknown parameter " << argv[argi] << " specified on command line.\n"; return; } } else { filenames.push_back(argv[argi]); } ++argi; } // We have to 'lie' about argc to create(..), because of a bug in older // gtkmms. int altArgc = 1; RFIGuiController controller; Glib::RefPtr app; std::unique_ptr window; if (interactive) { app = Gtk::Application::create("", Gio::Application::Flags::HANDLES_OPEN); SetupIconTheme(argv[0]); window = std::make_unique(&controller); window->present(); } try { if (!filenames.empty()) { if (interactive) { window->OpenPaths(filenames); } else { Pango::init(); MSOptions options; options.ioMode = DirectReadMode; options.dataColumnName = dataColumnName; options.combineSPWs = false; options.intervalStart = intervalStart; options.intervalEnd = intervalEnd; options.baselineIntegration.enable = false; controller.OpenMS(filenames, options); } } if (!savedBaselines.empty()) { imagesets::IndexableSet* imageSet = dynamic_cast(&controller.GetImageSet()); if (imageSet == nullptr) throw std::runtime_error( "Option -save-baseline can only be used for measurement sets.\n"); for (const SavedBaseline& b : savedBaselines) { auto index = imageSet->Index(b.a1Index, b.a2Index, b.bandIndex, b.sequenceIndex); if (!index) throw std::runtime_error("Baseline not found!"); controller.SetImageSetIndex(*index); StdOutReporter reporter; controller.LoadCurrentTFDataAsync(reporter); controller.LoadCurrentTFDataFinish(true); MaskedHeatMap& plot = controller.TFController().Plot(); plot.SetShowOriginalMask(plotFlags); plot.SetShowXAxisDescription(true); plot.SetShowYAxisDescription(true); plot.SetShowZAxisDescription(true); plot.SaveByExtension(b.filename, 800, 480); } } if (interactive) { app->signal_startup().connect( [app, win = window.get()] { app->add_window(*win); }); app->run(altArgc, argv); } } catch (const std::exception& e) { Logger::Error << "\n" "==========\n" "An unhandled exception occured while executing RFIGui. " "The error is:\n" << e.what() << '\n'; } } int main(int argc, char* argv[]) { run(argc, argv); Glib::Error::register_cleanup(); Glib::wrap_register_cleanup(); return 0; } aoflagger-v3.5.1/applications/aoqplot.cpp0000664000175000017500000001220415106657110016631 0ustar oleole#include "../aoqplot/aoqplotwindow.h" #include "../aoqplot/controllers/aoqplotcontroller.h" #include #include #include #include "../util/logger.h" #include "../rfigui/initialization.h" void SelectFile(AOQPlotWindow& window, std::unique_ptr& dialog, std::function callback) { dialog = std::make_unique(window, "Open observation set"); auto& fileDialog = static_cast(*dialog); fileDialog.add_button("_Cancel", Gtk::ResponseType::CANCEL); fileDialog.add_button("_Open", Gtk::ResponseType::OK); const Glib::RefPtr filter = Gtk::FileFilter::create(); filter->set_name("Observation sets (*.{vds,gds,ref,MS})"); filter->add_pattern("*.vds"); filter->add_pattern("*.gds"); filter->add_pattern("*.gvds"); filter->add_pattern("*.ref"); filter->add_pattern("*.MS"); fileDialog.add_filter(filter); fileDialog.signal_response().connect( [d = &fileDialog, callback](int response) { if (response == Gtk::ResponseType::OK) { const std::string filename = d->get_file()->get_path(); callback(filename); } }); fileDialog.show(); } int main(int argc, char* argv[]) { // We have to 'lie' about argc to create(..), because of a bug in older // gtkmms. int altArgc = 1; AOQPlotController controller; std::unique_ptr window; bool openGUI = true; int argi = 1; std::vector savedPlots; std::vector files; while (argi < argc) { if (argv[argi][0] == '-') { std::string p; if (argv[argi][1] == '-') p = &argv[argi][2]; else p = &argv[argi][1]; if (p == "help" || p == "h") { Logger::Info << "Syntax: aoqplot [] []\n\n" " can be a measurement set for opening a single " "observation.\n" "To get statistics for a (remote) observation consisting of " "multiple measurement\n" "sets, specify a measurement set specifier instead (generally a " ".ref, .vds\n" ".gvds or .gds file).\n" "\n" "Options can be:\n" "-help\n" " Show syntax help.\n" "-version\n" " Print version info and exit.\n" "-v\n" " Verbose logging.\n" "-save [filename prefix] [statistic name]\n" " Save every plot for the given kind of statistic as a PDF " "file. This\n" " will prevent the GUI from opening. You can repeat this " "parameter to save\n" " multiple kinds at once. A list of allowed names can be " "retrieved with\n" " 'aoquality liststats'. Some common ones are: " "StandardDeviation, Variance, Mean,\n" " RFIPercentage, RFIRatio, Count.\n" "\n" "AOQPlot is part of the AOFlagger software package, written by " "André Offringa\n" " (offringa@gmail.com). This AOQPlot belongs to AOFlagger " << AOFLAGGER_VERSION_STR << " (" << AOFLAGGER_VERSION_DATE_STR << ")\n"; return 0; } else if (p == "save") { AOQPlotController::PlotSavingData newPlot; newPlot.filenamePrefix = argv[argi + 1]; newPlot.statisticKind = QualityTablesFormatter::NameToKind(argv[argi + 2]); argi += 2; openGUI = false; savedPlots.push_back(newPlot); } else if (p == "version") { Logger::Info << "AOQplot " << AOFLAGGER_VERSION_STR << " (" << AOFLAGGER_VERSION_DATE_STR << ")\n"; return 0; } else if (p == "v") { Logger::SetVerbosity(Logger::VerboseVerbosity); } else { Logger::Error << "Bad parameter specified: " << argv[argi] << '\n'; return 1; } } else { files.push_back(argv[argi]); } ++argi; } std::unique_ptr dialog; if (openGUI) { const Glib::RefPtr app = Gtk::Application::create("", Gio::Application::Flags::HANDLES_OPEN); SetupIconTheme(argv[0]); window = std::make_unique(&controller); window->show(); if (files.empty()) { std::string filename; SelectFile(*window, dialog, [w = window.get()](const std::string& filename) { std::vector file{filename}; w->Open(file); }); } else { window->Open(files); } app->signal_startup().connect( [app, win = window.get()] { app->add_window(*win); }); app->run(altArgc, argv); } else { Pango::init(); if (files.empty()) { Logger::Error << "No observation specified.\n"; return 1; } controller.ReadStatistics(files); for (const AOQPlotController::PlotSavingData& plot : savedPlots) { controller.Save(plot, 640, 480); } } return 0; } aoflagger-v3.5.1/applications/aoflagger.cpp0000664000175000017500000002111514752462134017110 0ustar oleole#include "../aoluarunner/options.h" #include "../structures/types.h" #include "../util/logger.h" #include "../util/stopwatch.h" #include "../util/numberlist.h" #include "../aoluarunner/runner.h" #include #include #include #include #include #include #define RETURN_SUCCESS 0 #define RETURN_CMDLINE_ERROR 10 #define RETURN_STRATEGY_PARSE_ERROR 20 #define RETURN_UNHANDLED_EXCEPTION 30 void checkRelease() { #ifndef NDEBUG Logger::Warn << "This version of the AOFlagger has been compiled as DEBUG " "version! (NDEBUG was not defined)\n" << "For better performance, recompile it as a RELEASE.\n\n"; #endif } void generalInfo() { Logger::Info << "AOFlagger " << AOFLAGGER_VERSION_STR << " (" << AOFLAGGER_VERSION_DATE_STR << ") command line application\n" "Author: André Offringa (offringa@gmail.com)\n\n"; } int main(int argc, char** argv) { if (argc == 1) { generalInfo(); Logger::Error << "This program will execute a Lua flagging script that can be " "created with the RFI gui\n" "and executes it on one or several observations.\n\n" "Usage: " << argv[0] << " [options] [ [..]]" << R"( -v will produce verbose output -j overrides the number of threads specified in the strategy (default: one thread for each CPU core) -strategy Specifies a customized strategy. -direct-read Will perform the slowest IO but will always work. -indirect-read Will reorder the measurement set before starting, which is normally faster but requires free disk space to reorder the data to. -memory-read Will read the entire measurement set in memory. This is the fastest, but requires much memory. -auto-read-mode Will select either memory or direct mode based on available memory (default). -skip-flagged Will skip an ms if it has already been processed by AOFlagger according to its HISTORY table. -uvw Reads uvw values (some exotic strategies require these). -column Specify column to flag. -interval Only process the specified timesteps. Indices are zero indexed, and the end is exclusive, such that -interval 10 20 selects 10, 11, ... 19. -chunk-size This will split the set into intervals with the given maximum size, and flag each interval independently. This lowers the amount of memory required. The flagger has slightly less information per interval, but for a size of 1000 timesteps there is no noticable difference. With a size of 100 the difference is mostly not problematic either. In some cases, splitting the data increases accuracy, in particular when the statistics in the set change significantly over time (e.g. rising Galaxy). -bands Comma separated list of (zero-indexed) band ids to process. -fields Comma separated list of (zero-indexed) field ids to process. -baselines < all / cross / auto > Run the strategy on the given baseline types. The default is to run the strategy on all cross-correlation baselines. This parameter has no effect for single-dish observations. -combine-spws Join all SPWs together in frequency direction before flagging. -preamble Runs the specified Lua statement before starting to flag. This is typically used to define a variable, e.g. -preamble "bandpassfile = mybandpass.txt". -concatenate-frequency Reads all obs arguments and processes them as one measurement set. Every obs argument contains one band the same measurement; meaning the other metadata of the measurement sets is identical. This tool supports the Casacore measurement set, the SDFITS and Filterbank formats and some more. See the documentation for support of other file types. )"; checkRelease(); return RETURN_CMDLINE_ERROR; } Options options; size_t parameterIndex = 1; while (parameterIndex < (size_t)argc && argv[parameterIndex][0] == '-') { std::string flag(argv[parameterIndex] + 1); // If "--" was used, strip another dash if (!flag.empty() && flag[0] == '-') flag = flag.substr(1); if (flag == "j" && parameterIndex < (size_t)(argc - 1)) { ++parameterIndex; options.threadCount = atoi(argv[parameterIndex]); } else if (flag == "v") { options.logVerbosity = Logger::VerboseVerbosity; } else if (flag == "version") { Logger::Info << "AOFlagger " << AOFLAGGER_VERSION_STR << " (" << AOFLAGGER_VERSION_DATE_STR << ")\n"; return 0; } else if (flag == "direct-read") { options.readMode = DirectReadMode; } else if (flag == "reorder" || flag == "indirect-read") { options.readMode = ReorderingReadMode; if (flag == "indirect-read") Logger::Warn << "WARNING: Parameter '-indirect-read' is deprecated, " "use '-reorder' instead.\n"; } else if (flag == "memory-read") { options.readMode = MemoryReadMode; } else if (flag == "auto-read-mode") { options.readMode = AutoReadMode; } else if (flag == "strategy") { parameterIndex++; options.strategyFilename = argv[parameterIndex]; } else if (flag == "skip-flagged") { options.skipFlagged = true; } else if (flag == "uvw") { options.readUVW = true; } else if (flag == "column") { parameterIndex++; options.dataColumn = std::string(argv[parameterIndex]); } else if (flag == "bands") { ++parameterIndex; NumberList::ParseIntList(argv[parameterIndex], options.bands); } else if (flag == "fields") { ++parameterIndex; NumberList::ParseIntList(argv[parameterIndex], options.fields); } else if (flag == "combine-spws") { options.combineSPWs = true; } else if (flag == "concatenate-frequency") { options.concatenateFrequency = true; } else if (flag == "preamble") { ++parameterIndex; options.preamble.emplace_back(argv[parameterIndex]); } else if (flag == "interval") { options.startTimestep = atoi(argv[parameterIndex + 1]); options.endTimestep = atoi(argv[parameterIndex + 2]); parameterIndex += 2; } else if (flag == "chunk-size" || flag == "max-interval-size") { ++parameterIndex; options.chunkSize = atoi(argv[parameterIndex]); } else if (flag == "baselines") { ++parameterIndex; const std::string bTypes = argv[parameterIndex]; if (bTypes == "all") { options.baselineSelection = BaselineSelection::All; } else if (bTypes == "cross") { options.baselineSelection = BaselineSelection::CrossCorrelations; } else if (bTypes == "auto") { options.baselineSelection = BaselineSelection::AutoCorrelations; } else { Logger::Error << "Incorrect usage; baselines parameter should be set " "to 'all', 'cross' or 'auto'.\n"; return RETURN_CMDLINE_ERROR; } } else { Logger::Error << "Incorrect usage; parameter \"" << argv[parameterIndex] << "\" not understood.\n"; return RETURN_CMDLINE_ERROR; } ++parameterIndex; } try { Logger::SetVerbosity( options.logVerbosity.value_or(Logger::NormalVerbosity)); generalInfo(); checkRelease(); const Stopwatch watch(true); std::stringstream commandLineStr; commandLineStr << argv[0]; for (int i = 1; i < argc; ++i) { commandLineStr << " \"" << argv[i] << '\"'; } options.commandLine = commandLineStr.str(); for (int i = parameterIndex; i < argc; ++i) options.filenames.emplace_back(argv[i]); const std::filesystem::path strategyPath = options.strategyFilename; if (boost::to_lower_copy(strategyPath.extension().string()) == ".rfis") { Logger::Error << "An old .rfis file was specified. AOFlagger version 3 " "supports only Lua scripts and can\n" "not run the old .rfis-style files. Example Lua " "strategies can be found in the aoflagger\n" "source directory under data/strategies.\n"; return RETURN_CMDLINE_ERROR; } Runner runner(options); runner.Run(); Logger::Debug << "Time: " << watch.ToString() << "\n"; return RETURN_SUCCESS; } catch (std::exception& exception) { std::cerr << "An unhandled exception occured: " << exception.what() << '\n' << "If you think this is a bug, please contact offringa@gmail.com\n"; return RETURN_UNHANDLED_EXCEPTION; } } aoflagger-v3.5.1/applications/aoquality.cpp0000664000175000017500000003171314752462134017176 0ustar oleole#include #include "../util/plot.h" #include "../quality/operations.h" #include namespace { std::vector AsVector(char* argv[], size_t start, size_t argc) { std::vector result; for (size_t i = start; i != argc; ++i) result.emplace_back(argv[i]); return result; } void printSyntax(std::ostream& stream, char* argv[]) { stream << "Syntax: " << argv[0] << " [options]\n\n" "Possible actions:\n" "\thelp - Get more info about an action (usage: '" << argv[0] << " help ')\n" "\tcollect - Processes the entire measurement set, collects " "the statistics\n" "\t and writes them in the quality tables.\n" "\tcombine - Combine several tables.\n" "\thistogram - Various histogram actions.\n" "\tliststats - Display a list of possible statistic kinds.\n" "\tquery_a - Query per antenna.\n" "\tquery_b - Query per baseline.\n" "\tquery_t - Query per time step.\n" "\tquery_f - Query per frequency.\n" "\tquery_fr - Query a frequency range\n" "\tquery_g - Query single global statistic.\n" "\tremove - Remove all quality tables.\n" "\tsummarize - Give a summary of the statistics currently in the " "quality tables.\n" "\tsummarizerfi- Give a summary of the rfi statistics.\n" "\n\n" "A few actions take a statistic kind. Some common statistic kinds " "are: StandardDeviation,\n" "Variance, Mean, RFIPercentage, RFIRatio, Count. These are case " "sensitive. Run 'aoquality liststats' for a full list.\n"; } void HistogramAction(const std::string& filename, const std::string& query, const char* dataColumnName) { if (query == "rfislope") { quality::PrintRfiSlope(filename); } else if (query == "rfislope-per-baseline") { quality::PrintRfiSlopePerBaseline(filename, dataColumnName); } else if (query == "remove") { quality::RemoveHistogram(filename); } else { std::cerr << "Unknown histogram command: " << query << "\n"; } } } // namespace int main(int argc, char* argv[]) { #ifdef HAS_LOFARSTMAN register_lofarstman(); #endif // HAS_LOFARSTMAN if (argc < 2) { printSyntax(std::cerr, argv); return -1; } else { const std::string action = argv[1]; if (action == "help") { if (argc != 3) { printSyntax(std::cout, argv); } else { const std::string helpAction = argv[2]; if (helpAction == "help") { printSyntax(std::cout, argv); } else if (helpAction == "collect") { std::cout << "Syntax: " << argv[0] << " collect [-d [column]/-tf/-h] [quack timesteps] [list " "of antennae]\n\n" "The collect action will go over a whole measurement set and " "\n" "collect the default statistics. It will write the results in " "the \n" "quality subtables of the main measurement set.\n\n" "Currently, the default statistics are:\n" "\tRFIRatio, Count, Mean, SumP2, DCount, DMean, DSumP2.\n" "The subtables that will be updated are:\n" "\tQUALITY_KIND_NAME, QUALITY_TIME_STATISTIC,\n" "\tQUALITY_FREQUENCY_STATISTIC and " "QUALITY_BASELINE_STATISTIC.\n\n"; } else if (helpAction == "summarize") { std::cout << "Syntax: " << argv[0] << " summarize \n\n" "Gives a summary of the statistics in the measurement set.\n"; } else if (helpAction == "query_a") { std::cout << "Syntax: " << argv[0] << " query_a \n\n" "Prints the given statistic for each antenna.\n"; } else if (helpAction == "query_b") { std::cout << "Syntax: " << argv[0] << " query_b \n\n" "Prints the given statistic for each baseline.\n"; } else if (helpAction == "query_t") { std::cout << "Syntax: " << argv[0] << " query_t \n\n" "Print the given statistic for each time step.\n"; } else if (helpAction == "query_f") { std::cout << "Syntax: " << argv[0] << " query_f \n\n" "Print the given statistic for each frequency.\n"; } else if (helpAction == "query_g") { std::cout << "Syntax " << argv[0] << " query_g \n\n" "Print the given statistic for this measurement set.\n"; } else if (helpAction == "combine") { std::cout << "Syntax: " << argv[0] << " combine [ [ ..]]\n\n" "This will read all given input measurement sets, " "combine the statistics and \n" "write the results to a target measurement set. The " "target measurement set should\n" "not exist beforehand.\n"; } else if (helpAction == "histogram") { std::cout << "Syntax: " << argv[0] << " histogram ]\n\n" "Query can be:\n" "\trfislope - performs linear regression on the part of " "the histogram that should contain the RFI.\n" "\t Reports one value per polarisation.\n"; } else if (helpAction == "remove") { std::cout << "Syntax: " << argv[0] << " remove [ms]\n\n" "This will completely remove all quality tables from " "the measurement set.\n"; } else { std::cerr << "Unknown action specified in help.\n"; return -1; } } } else if (action == "liststats") { quality::ListStatistics(); } else if (action == "collect") { if (argc < 3) { std::cerr << "collect actions needs one or two parameters (the " "measurement set)\n"; return -1; } else { int argi = 2; bool histograms = false, timeFrequency = false; const char* dataColumnName = "DATA"; size_t intervalStart = 0, intervalEnd = 0; while (argi < argc && argv[argi][0] == '-') { const std::string p = &argv[argi][1]; if (p == "h") { histograms = true; } else if (p == "d") { ++argi; dataColumnName = argv[argi]; } else if (p == "tf") { timeFrequency = true; } else if (p == "interval") { intervalStart = atoi(argv[argi + 1]); intervalEnd = atoi(argv[argi + 2]); argi += 2; } else { throw std::runtime_error( "Bad parameter given to aoquality collect"); } ++argi; } const std::string filename = argv[argi]; size_t flaggedTimesteps = 0; ++argi; std::set flaggedAntennae; if (argi != argc) { flaggedTimesteps = atoi(argv[argi]); ++argi; while (argi != argc) { flaggedAntennae.insert(atoi(argv[argi])); ++argi; } } Collector::CollectingMode mode; if (histograms) mode = Collector::CollectHistograms; else if (timeFrequency) mode = Collector::CollectTimeFrequency; else mode = Collector::CollectDefault; quality::CollectStatistics(filename, mode, flaggedTimesteps, std::move(flaggedAntennae), dataColumnName, intervalStart, intervalEnd); } } else if (action == "combine") { if (argc < 3) { std::cerr << "combine actions needs at least one parameter: aoquality " "combine [ ...]\n"; return -1; } else { const std::string outFilename = argv[2]; quality::CombineStatistics(outFilename, AsVector(argv, 3, argc)); } } else if (action == "histogram") { if (argc < 4) { std::cerr << "histogram actions needs at least two parameters (the query and " "the measurement set)\n"; return -1; } else { HistogramAction(argv[3], argv[2], "DATA"); } } else if (action == "summarize") { if (argc < 3) { std::cerr << "summarize actions needs at least one parameter (the " "measurement set)\n"; return -1; } else { quality::PrintSummary(AsVector(argv, 2, argc)); } } else if (action == "summarizerfi") { if (argc < 3) { std::cerr << "summarizerfi actions needs at least one parameter (the " "measurement set)\n"; return -1; } else { quality::PrintRfiSummary(AsVector(argv, 2, argc)); } } else if (action == "query_g") { if (argc < 4) { std::cerr << "Syntax for query global stat: 'aoquality query_g " " [ ...]'\n"; return -1; } else { quality::PrintGlobalStatistic(argv[2], AsVector(argv, 3, argc)); } } else if (action == "query_a") { if (argc < 4) { std::cerr << "Syntax for query antennas: 'aoquality query_a " " [ ...]'\n"; return -1; } else { quality::PrintPerAntennaStatistics(argv[2], AsVector(argv, 3, argc)); return 0; } } else if (action == "query_b") { if (argc < 4) { std::cerr << "Syntax for query baselines: 'aoquality query_b " " [ ...]'\n"; return -1; } else { quality::PrintPerBaselineStatistics(argv[2], AsVector(argv, 3, argc)); } } else if (action == "query_f") { if (argc < 4) { std::cerr << "Syntax for query times: 'aoquality query_t [options] " " [ ...]'\n" "Options:\n" " -downsample \n" " Average down the statistics in frequency to the " "given nr of bins.\n"; return -1; } else { size_t argi = 2; std::optional downsample; while (argv[argi][0] == '-') { const std::string p(&argv[argi][1]); if (p == "downsample") { ++argi; downsample = std::atoi(argv[argi]); } else { throw std::runtime_error("Invalid parameter: " + p); } ++argi; } quality::PrintPerFrequencyStatistics( argv[argi], AsVector(argv, argi + 1, argc), downsample); return 0; } } else if (action == "query_fr") { if (argc == 5) { const std::string range = argv[4]; if (range == "DVB4") { quality::PrintFrequencyRangeStatistic( argv[2], AsVector(argv, 3, argc), 167, 174); } else if (range == "DVB5") { quality::PrintFrequencyRangeStatistic( argv[2], AsVector(argv, 3, argc), 174, 181); } else if (range == "DVB6") { quality::PrintFrequencyRangeStatistic( argv[2], AsVector(argv, 3, argc), 181, 188); } else if (range == "DVB7") { quality::PrintFrequencyRangeStatistic( argv[2], AsVector(argv, 3, argc), 188, 195); } else { std::cerr << "Syntax for query times: 'aoquality query_fr " " '\n"; return -1; } return 0; } else if (argc == 6) { quality::PrintFrequencyRangeStatistic(argv[2], {argv[3]}, atof(argv[4]), atof(argv[5])); return 0; } else { std::cerr << "Syntax for query frequency range: 'aoquality query_fr " " " " '\n"; return -1; } } else if (action == "query_t") { if (argc < 4) { std::cerr << "Syntax for query times: 'aoquality query_t " "[ ...]'\n"; return -1; } else { quality::PrintPerTimeStatistics(argv[2], AsVector(argv, 3, argc)); return 0; } } else if (action == "remove") { if (argc != 3) { std::cerr << "Syntax for removing quality tables: 'aoquality remove '\n"; return -1; } else { quality::RemoveStatistics(argv[2]); return 0; } } else { std::cerr << "Unknown action '" << action << "'.\n\n"; printSyntax(std::cerr, argv); return -1; } return 0; } } aoflagger-v3.5.1/applications/badstations.cpp0000664000175000017500000001602014752462134017473 0ustar oleole#include "../algorithms/antennaselector.h" #include "../algorithms/baselineselector.h" #include "../structures/msmetadata.h" #include "../quality/defaultstatistics.h" #include "../quality/histogramcollection.h" #include "../quality/qualitytablesformatter.h" #include "../quality/statisticscollection.h" #include "../quality/statisticsderivator.h" #include "../quality/histogramtablesformatter.h" #include #include #include #include using algorithms::AntennaSelector; using algorithms::BaselineSelector; StatisticsCollection load(const std::string& filename, std::vector& antennae) { StatisticsCollection statisticsCollection; const HistogramCollection histogramCollection; const MSMetaData ms(filename); const unsigned polarizationCount = ms.PolarizationCount(); statisticsCollection.SetPolarizationCount(polarizationCount); QualityTablesFormatter qualityData(filename); statisticsCollection.Load(qualityData); const unsigned antennaCount = ms.AntennaCount(); for (unsigned a = 0; a < antennaCount; ++a) antennae.push_back(ms.GetAntennaInfo(a)); return statisticsCollection; } std::set detectRFIPercentage(const char* filename) { std::vector antennae; StatisticsCollection statisticsCollection = load(filename, antennae); BaselineSelector selector; selector.SetUseLog(true); statisticsCollection.IntegrateBaselinesToOneChannel(); const BaselineStatisticsMap& baselineMap = statisticsCollection.BaselineStatistics(); const std::vector> list = baselineMap.BaselineList(); for (std::vector>::const_iterator i = list.begin(); i != list.end(); ++i) { const unsigned a1 = i->first, a2 = i->second; DefaultStatistics statistic = baselineMap.GetStatistics(a1, a2); selector.Add(statistic, antennae[a1], antennae[a2]); } std::vector markedBaselines; std::set badStations; selector.Search(markedBaselines); selector.ImplyStations(markedBaselines, 0.3, badStations); std::cout << "List of " << badStations.size() << " bad stations:\n"; for (const size_t ant : badStations) { std::cout << antennae[ant].name << " (" << ant << ")\n"; } return std::set(badStations.begin(), badStations.end()); } std::set detectStddev(const char* filename) { std::vector antennae; const StatisticsCollection statisticsCollection = load(filename, antennae); AntennaSelector selector; std::vector badStations = selector.Run(statisticsCollection); std::cout << "List of " << badStations.size() << " bad stations:\n"; for (const size_t ant : badStations) { std::cout << antennae[ant].name << " (" << ant << ")\n"; } return std::set(badStations.begin(), badStations.end()); } void flagAntennas(const char* filename, const std::set& antennae) { casacore::MeasurementSet ms(filename, casacore::Table::Update); /** * Read some meta data from the measurement set */ const casacore::MSSpectralWindow spwTable = ms.spectralWindow(); const size_t spwCount = spwTable.nrow(); if (spwCount != 1) throw std::runtime_error("Set should have exactly one spectral window"); const casacore::ScalarColumn numChanCol( spwTable, casacore::MSSpectralWindow::columnName( casacore::MSSpectralWindowEnums::NUM_CHAN)); const size_t channelCount = numChanCol.get(0); if (channelCount == 0) throw std::runtime_error("No channels in set"); const casacore::ScalarColumn ant1Column( ms, ms.columnName(casacore::MSMainEnums::ANTENNA1)); const casacore::ScalarColumn ant2Column( ms, ms.columnName(casacore::MSMainEnums::ANTENNA2)); casacore::ArrayColumn flagsColumn( ms, ms.columnName(casacore::MSMainEnums::FLAG)); if (ms.nrow() == 0) throw std::runtime_error("Table has no rows (no data)"); const casacore::IPosition flagsShape = flagsColumn.shape(0); const casacore::Array flags(flagsShape, true); std::cout << "Flagging... " << std::flush; /** * Flag */ size_t crossCount = 0, autoCount = 0; for (size_t rowIndex = 0; rowIndex != ms.nrow(); ++rowIndex) { // Selected? if (antennae.find(ant1Column.get(rowIndex)) != antennae.end() || antennae.find(ant2Column.get(rowIndex)) != antennae.end()) { if (ant1Column.get(rowIndex) == ant2Column.get(rowIndex)) ++autoCount; else ++crossCount; flagsColumn.put(rowIndex, flags); } } std::cout << "DONE (selected " << crossCount << " cross- and " << autoCount << " auto-correlated timesteps)\n"; } void printSyntax(std::ostream& stream, char* argv[]) { stream << "The executable 'badstations' will give a list of stations that " "are outliers\n" "according to the RFI statistics.\n" "\n" "Syntax: badstations [options] \n" "\n" "Options:\n" "-flag\n" " Will not only show statistics, but also flag the antennas that " "are found\n" " to be bad in the measurement set.\n" " (this only works if the given filename is a measurement set, " "not a .ref).\n" "-method \n" " Select detection method. Method 'stddev' is the default, and " "simply detects\n" " stations with an outlyer standard deviation. Method " "'percentage' detects\n" " outliers based on the percentage RFI statistic, taking into " "account that\n" " short baselines often see fewer RFI, by fitting a curve to the " "statistic\n" " as a function of baseline.\n"; } int main(int argc, char* argv[]) { #ifdef HAS_LOFARSTMAN register_lofarstman(); #endif // HAS_LOFARSTMAN int argi = 1; bool doFlag = false; enum Method { StddevMethod, RFIPercentangeMethod } method = StddevMethod; while (argi < argc && argv[argi][0] == '-') { const std::string p(argv[argi] + 1); if (p == "flag") { doFlag = true; } else if (p == "method") { ++argi; const std::string m = argv[argi]; if (m == "stddev") method = StddevMethod; else if (m == "percentage") method = RFIPercentangeMethod; else throw std::runtime_error("Unknown method given"); } else { throw std::runtime_error("Unknown parameter"); } ++argi; } if (argi >= argc) { printSyntax(std::cerr, argv); return -1; } else { const char* filename = argv[argi]; std::set badAntennas; switch (method) { case StddevMethod: badAntennas = detectStddev(filename); break; case RFIPercentangeMethod: badAntennas = detectRFIPercentage(filename); break; } if (doFlag) flagAntennas(filename, badAntennas); return 0; } } aoflagger-v3.5.1/.clang-format0000664000175000017500000001022214752462134014337 0ustar oleole--- Language: Cpp # BasedOnStyle: Google AccessModifierOffset: -1 AlignAfterOpenBracket: Align AlignConsecutiveMacros: false AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false AlignEscapedNewlines: Left AlignOperands: true AlignTrailingComments: true AllowAllArgumentsOnNextLine: true AllowAllConstructorInitializersOnNextLine: true AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: All AllowShortLambdasOnASingleLine: All AllowShortIfStatementsOnASingleLine: WithoutElse AllowShortLoopsOnASingleLine: true AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: Yes BinPackArguments: true BinPackParameters: true BraceWrapping: AfterCaseLabel: false AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: false AfterNamespace: false AfterObjCDeclaration: false AfterStruct: false AfterUnion: false AfterExternBlock: false BeforeCatch: false BeforeElse: false IndentBraces: false SplitEmptyFunction: true SplitEmptyRecord: true SplitEmptyNamespace: true BreakBeforeBinaryOperators: None BreakBeforeBraces: Attach BreakBeforeInheritanceComma: false BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false BreakConstructorInitializers: BeforeColon BreakAfterJavaFieldAnnotations: false BreakStringLiterals: true ColumnLimit: 80 CommentPragmas: '^ IWYU pragma:' CompactNamespaces: false ConstructorInitializerAllOnOneLineOrOnePerLine: true ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: false DisableFormat: false ExperimentalAutoDetectBinPacking: false FixNamespaceComments: true ForEachMacros: - foreach - Q_FOREACH - BOOST_FOREACH IncludeBlocks: Regroup IncludeCategories: - Regex: '^' Priority: 2 - Regex: '^<.*\.h>' Priority: 1 - Regex: '^<.*' Priority: 2 - Regex: '.*' Priority: 3 IncludeIsMainRegex: '([-_](test|unittest))?$' IndentCaseLabels: true IndentPPDirectives: None IndentWidth: 2 IndentWrappedFunctionNames: false JavaScriptQuotes: Leave JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: false MacroBlockBegin: '' MacroBlockEnd: '' MaxEmptyLinesToKeep: 1 NamespaceIndentation: None ObjCBinPackProtocolList: Never ObjCBlockIndentWidth: 2 ObjCSpaceAfterProperty: false ObjCSpaceBeforeProtocolList: true PenaltyBreakAssignment: 2 PenaltyBreakBeforeFirstCallParameter: 1 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left RawStringFormats: - Language: Cpp Delimiters: - cc - CC - cpp - Cpp - CPP - 'c++' - 'C++' CanonicalDelimiter: '' BasedOnStyle: google - Language: TextProto Delimiters: - pb - PB - proto - PROTO EnclosingFunctions: - EqualsProto - EquivToProto - PARSE_PARTIAL_TEXT_PROTO - PARSE_TEST_PROTO - PARSE_TEXT_PROTO - ParseTextOrDie - ParseTextProtoOrDie CanonicalDelimiter: '' BasedOnStyle: google ReflowComments: true SortIncludes: false SortUsingDeclarations: true SpaceAfterCStyleCast: false SpaceAfterLogicalNot: false SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true SpaceBeforeCpp11BracedList: false SpaceBeforeCtorInitializerColon: true SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements SpaceBeforeRangeBasedForLoopColon: true SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 2 SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false Standard: c++17 StatementMacros: - Q_UNUSED - QT_REQUIRE_VERSION TabWidth: 8 UseTab: Never ... aoflagger-v3.5.1/quality/0000775000175000017500000000000015146315735013461 5ustar oleoleaoflagger-v3.5.1/quality/collector.h0000664000175000017500000000360214752462134015617 0ustar oleole#ifndef COLLECTOR_H #define COLLECTOR_H #include #include #include #include #include #include #include class HistogramCollection; class StatisticsCollection; class ProgressListener; class Collector { public: enum CollectingMode { CollectDefault, CollectHistograms, CollectTimeFrequency }; Collector(); ~Collector(); void Collect(const std::string& filename, StatisticsCollection& statisticsCollection, HistogramCollection& histogramCollection, ProgressListener& progressListener); void SetMode(CollectingMode mode) { _mode = mode; } void SetDataColumn(const std::string& dataColumnName) { _dataColumnName = dataColumnName; } void SetFlaggedTimesteps(size_t flaggedTimesteps) { _flaggedTimesteps = flaggedTimesteps; } void SetInterval(size_t start, size_t end) { _intervalStart = start; _intervalEnd = end; } void SetFlaggedAntennae(std::set&& flaggedAntennae) { _flaggedAntennae = flaggedAntennae; } private: struct Work { std::vector>> samples; std::vector> isRFI; size_t antenna1Index, antenna2Index, bandIndex, timestepIndex, nChan; double time; bool hasFlaggedAntenna; }; void process(aocommon::Lane* workLane, StatisticsCollection* stats, size_t polarizationCount); std::mutex _mutex; CollectingMode _mode; std::string _dataColumnName; size_t _intervalStart, _intervalEnd; size_t _flaggedTimesteps; std::set _flaggedAntennae; aocommon::UVector _correlatorFlags; aocommon::UVector _correlatorFlagsForBadAntenna; StatisticsCollection* _statisticsCollection; HistogramCollection* _histogramCollection; std::vector _threadStats; }; #endif aoflagger-v3.5.1/quality/operations.cpp0000664000175000017500000005473514752462134016364 0ustar oleole#include "operations.h" #include #include #include #include "collector.h" #include "combine.h" #include "defaultstatistics.h" #include "histogramcollection.h" #include "qualitytablesformatter.h" #include "statisticscollection.h" #include "statisticsderivator.h" #include "../util/progress/stdoutreporter.h" #include "../structures/msmetadata.h" namespace quality { namespace { void PrintStatistics(std::complex* complexStat, unsigned count) { if (count != 1) std::cout << '['; if (count > 0) std::cout << complexStat[0].real() << " + " << complexStat[0].imag() << 'i'; for (unsigned p = 1; p < count; ++p) { std::cout << ", " << complexStat[p].real() << " + " << complexStat[p].imag() << 'i'; } if (count != 1) std::cout << ']'; } void PrintStatistics(unsigned long* stat, unsigned count) { if (count != 1) std::cout << '['; if (count > 0) std::cout << stat[0]; for (unsigned p = 1; p < count; ++p) { std::cout << ", " << stat[p]; } if (count != 1) std::cout << ']'; } void PrintStatistics(const DefaultStatistics& statistics) { std::cout << "Count="; PrintStatistics(statistics.count, statistics.PolarizationCount()); std::cout << "\nSum="; PrintStatistics(statistics.sum, statistics.PolarizationCount()); std::cout << "\nSumP2="; PrintStatistics(statistics.sumP2, statistics.PolarizationCount()); std::cout << "\nDCount="; PrintStatistics(statistics.dCount, statistics.PolarizationCount()); std::cout << "\nDSum="; PrintStatistics(statistics.dSum, statistics.PolarizationCount()); std::cout << "\nDSumP2="; PrintStatistics(statistics.dSumP2, statistics.PolarizationCount()); std::cout << "\nRFICount="; PrintStatistics(statistics.rfiCount, statistics.PolarizationCount()); std::cout << '\n'; } void PrintRFISlopeForHistogram(const std::map& histogramMap, char polarizationSymbol, const AntennaInfo* antennae) { for (std::map::const_iterator i = histogramMap.begin(); i != histogramMap.end(); ++i) { const unsigned a1 = i->first.first, a2 = i->first.second; const Baseline baseline(antennae[a1], antennae[a2]); const double length = baseline.Distance(); const LogHistogram& histogram = *i->second; double start, end; histogram.GetRFIRegion(start, end); const double slope = histogram.NormalizedSlope(start, end); const double stddev = histogram.NormalizedSlopeStdError(start, end, slope); std::cout << polarizationSymbol << '\t' << a1 << '\t' << a2 << '\t' << length << '\t' << slope << '\t' << stddev << '\n'; } } } // namespace void ListStatistics() { for (int i = 0; i != QualityTablesFormatter::EndPlaceHolderStatistic; ++i) { const QualityTablesFormatter::StatisticKind kind = (QualityTablesFormatter::StatisticKind)i; std::cout << QualityTablesFormatter::KindToName(kind) << '\n'; } } void CollectStatistics(const std::string& filename, Collector::CollectingMode mode, size_t flaggedTimesteps, std::set&& flaggedAntennae, const char* dataColumnName, size_t intervalStart, size_t intervalEnd) { StatisticsCollection statisticsCollection; HistogramCollection histogramCollection; Collector collector; collector.SetDataColumn(dataColumnName); collector.SetInterval(intervalStart, intervalEnd); collector.SetMode(mode); collector.SetFlaggedAntennae(std::move(flaggedAntennae)); collector.SetFlaggedTimesteps(flaggedTimesteps); StdOutReporter reporter; collector.Collect(filename, statisticsCollection, histogramCollection, reporter); switch (mode) { case Collector::CollectDefault: case Collector::CollectTimeFrequency: { std::cout << "Writing quality tables..." << std::endl; QualityTablesFormatter qualityData(filename); statisticsCollection.Save(qualityData); } break; case Collector::CollectHistograms: { std::cout << "Writing histogram tables..." << std::endl; HistogramTablesFormatter histograms(filename); histogramCollection.Save(histograms); } break; } std::cout << "Done.\n"; } void CollectHistograms(const std::string& filename, HistogramCollection& histogramCollection, size_t flaggedTimesteps, std::set&& flaggedAntennae, const char* dataColumnName) { std::cout << "Collecting statistics...\n"; StatisticsCollection tempCollection; Collector collector; collector.SetDataColumn(dataColumnName); collector.SetMode(Collector::CollectHistograms); collector.SetFlaggedAntennae(std::move(flaggedAntennae)); collector.SetFlaggedTimesteps(flaggedTimesteps); StdOutReporter reporter; collector.Collect(filename, tempCollection, histogramCollection, reporter); } void PrintGlobalStatistic(const std::string& kindName, const std::vector& filenames) { const MSMetaData ms(filenames.front()); BandInfo band; if (ms.BandCount() != 0) band = ms.GetBandInfo(0); const QualityTablesFormatter::StatisticKind kind = QualityTablesFormatter::NameToKind(kindName); const quality::FileContents contents = quality::ReadAndCombine(filenames, false); const unsigned n_polarizations = contents.statistics_collection.PolarizationCount(); DefaultStatistics statistics(n_polarizations); contents.statistics_collection.GetGlobalCrossBaselineStatistics(statistics); const StatisticsDerivator derivator(contents.statistics_collection); const DefaultStatistics singlePol(statistics.ToSinglePolarization()); double start, end; if (band.channels.empty()) { start = 0.0; end = 0.0; } else { start = band.channels.begin()->frequencyHz; end = band.channels.rbegin()->frequencyHz; } std::cout << round(start / 10000.0) / 100.0 << '\t' << round(end / 10000.0) / 100.0 << '\t' << derivator.GetStatisticAmplitude(kind, singlePol, 0); for (unsigned p = 0; p < n_polarizations; ++p) { const long double val = derivator.GetStatisticAmplitude(kind, statistics, p); std::cout << '\t' << val; } std::cout << '\n'; } void PrintFrequencyRangeStatistic(const std::string& kindName, const std::vector& filenames, double startFreqMHz, double endFreqMHz) { const QualityTablesFormatter::StatisticKind kind = QualityTablesFormatter::NameToKind(kindName); const quality::FileContents contents = quality::ReadAndCombine(filenames, false); const unsigned n_polarizations = contents.statistics_collection.PolarizationCount(); DefaultStatistics statistics(n_polarizations); contents.statistics_collection.GetFrequencyRangeStatistics( statistics, startFreqMHz * 1e6, endFreqMHz * 1e6); const StatisticsDerivator derivator(contents.statistics_collection); const DefaultStatistics singlePol(statistics.ToSinglePolarization()); std::cout << startFreqMHz << '\t' << endFreqMHz << '\t' << derivator.GetStatisticAmplitude(kind, singlePol, 0); for (unsigned p = 0; p < n_polarizations; ++p) { const long double val = derivator.GetStatisticAmplitude(kind, statistics, p); std::cout << '\t' << val; } std::cout << '\n'; } void PrintPerBaselineStatistics(const std::string& kindName, const std::vector& filenames) { const QualityTablesFormatter::StatisticKind kind = QualityTablesFormatter::NameToKind(kindName); quality::FileContents contents = quality::ReadAndCombine(filenames, false); contents.statistics_collection.IntegrateBaselinesToOneChannel(); const std::vector>& baselines = contents.statistics_collection.BaselineStatistics().BaselineList(); const StatisticsDerivator derivator(contents.statistics_collection); std::cout << "ANTENNA1\tANTENNA2"; const unsigned n_polarizations = contents.statistics_collection.PolarizationCount(); for (unsigned p = 0; p < n_polarizations; ++p) std::cout << '\t' << kindName << "_POL" << p << "_R\t" << kindName << "_POL" << p << "_I"; std::cout << '\n'; for (std::vector>::const_iterator i = baselines.begin(); i != baselines.end(); ++i) { const unsigned antenna1 = i->first, antenna2 = i->second; std::cout << antenna1 << '\t' << antenna2; for (unsigned p = 0; p < n_polarizations; ++p) { const std::complex val = derivator.GetComplexBaselineStatistic(kind, antenna1, antenna2, p); std::cout << '\t' << val.real() << '\t' << val.imag(); } std::cout << '\n'; } } void PrintPerFrequencyStatistics(const std::string& kindName, const std::vector& filenames, std::optional downsample) { const QualityTablesFormatter::StatisticKind kind = QualityTablesFormatter::NameToKind(kindName); quality::FileContents contents = quality::ReadAndCombine(filenames, false); if (downsample) contents.statistics_collection.LowerFrequencyResolution(*downsample); const std::map& freqStats = contents.statistics_collection.FrequencyStatistics(); const StatisticsDerivator derivator(contents.statistics_collection); std::cout << "FREQUENCY"; const unsigned n_polarizations = contents.statistics_collection.PolarizationCount(); for (size_t p = 0; p < n_polarizations; ++p) std::cout << '\t' << kindName << "_POL" << p << "_R\t" << kindName << "_POL" << p << "_I"; std::cout << '\n'; for (const std::pair& freqStat : freqStats) { const double frequency = freqStat.first; std::cout << frequency * 1e-6; for (unsigned p = 0; p < n_polarizations; ++p) { const std::complex val = derivator.GetComplexStatistic(kind, freqStat.second, p); std::cout << '\t' << val.real() << '\t' << val.imag(); } std::cout << '\n'; } } void PrintPerTimeStatistics(const std::string& kindName, const std::vector& filenames) { const QualityTablesFormatter::StatisticKind kind = QualityTablesFormatter::NameToKind(kindName); quality::FileContents contents = quality::ReadAndCombine(filenames, false); contents.statistics_collection.IntegrateTimeToOneChannel(); const std::map& timeStats = contents.statistics_collection.TimeStatistics(); const StatisticsDerivator derivator(contents.statistics_collection); std::cout << "TIME"; const unsigned n_polarizations = contents.statistics_collection.PolarizationCount(); for (unsigned p = 0; p < n_polarizations; ++p) std::cout << '\t' << kindName << "_POL" << p << "_R\t" << kindName << "_POL" << p << "_I"; std::cout << '\n'; for (std::map::const_iterator i = timeStats.begin(); i != timeStats.end(); ++i) { const double time = i->first; std::cout << time; for (unsigned p = 0; p < n_polarizations; ++p) { const std::complex val = derivator.GetComplexStatistic(kind, i->second, p); std::cout << '\t' << val.real() << '\t' << val.imag(); } std::cout << '\n'; } } void PrintPerAntennaStatistics(const std::string& kindName, const std::vector& filenames) { const QualityTablesFormatter::StatisticKind kind = QualityTablesFormatter::NameToKind(kindName); quality::FileContents contents = quality::ReadAndCombine(filenames, false); contents.statistics_collection.IntegrateBaselinesToOneChannel(); const std::map stats = contents.statistics_collection.GetAntennaStatistics(); const StatisticsDerivator derivator(contents.statistics_collection); std::cout << "ANTENNA"; const unsigned n_polarizations = contents.statistics_collection.PolarizationCount(); for (unsigned p = 0; p < n_polarizations; ++p) std::cout << '\t' << kindName << "_POL" << p << "_R\t" << kindName << "_POL" << p << "_I"; std::cout << '\n'; for (const std::pair& s : stats) { const size_t antenna = s.first; std::cout << antenna; for (unsigned p = 0; p < n_polarizations; ++p) { const std::complex val = derivator.GetComplexStatistic(kind, s.second, p); std::cout << '\t' << val.real() << '\t' << val.imag(); } std::cout << '\n'; } } void PrintSummary(const std::vector& filenames) { const quality::FileContents contents = quality::ReadAndCombine(filenames, false); const unsigned n_polarizations = contents.statistics_collection.PolarizationCount(); DefaultStatistics statistics(n_polarizations); contents.statistics_collection.GetGlobalTimeStatistics(statistics); std::cout << "Time statistics: \n"; PrintStatistics(statistics); contents.statistics_collection.GetGlobalFrequencyStatistics(statistics); std::cout << "\nFrequency statistics: \n"; PrintStatistics(statistics); contents.statistics_collection.GetGlobalCrossBaselineStatistics(statistics); std::cout << "\nCross-correlated baseline statistics: \n"; PrintStatistics(statistics); const DefaultStatistics singlePolStat = statistics.ToSinglePolarization(); std::cout << "RFIPercentange: " << StatisticsDerivator::GetStatisticAmplitude( QualityTablesFormatter::RFIPercentageStatistic, singlePolStat, 0) << '\n'; contents.statistics_collection.GetGlobalAutoBaselineStatistics(statistics); std::cout << "\nAuto-correlated baseline: \n"; PrintStatistics(statistics); } void PrintRfiSummary(const std::vector& filenames) { const MSMetaData ms(filenames.front()); const BandInfo band = ms.GetBandInfo(0); const quality::FileContents contents = quality::ReadAndCombine(filenames, false); DefaultStatistics statistics( contents.statistics_collection.PolarizationCount()); contents.statistics_collection.GetGlobalCrossBaselineStatistics(statistics); const DefaultStatistics singlePolStat = statistics.ToSinglePolarization(); double startTime = contents.statistics_collection.TimeStatistics().begin()->first, endTime = contents.statistics_collection.TimeStatistics().rbegin()->first, startFreq = band.channels.begin()->frequencyHz, endFreq = band.channels.rbegin()->frequencyHz; std::cout.precision(16); std::cout << startTime << '\t' << endTime << '\t' << round(startFreq / 10000.0) / 100.0 << '\t' << round(endFreq / 10000.0) / 100.0 << '\t' << StatisticsDerivator::GetStatisticAmplitude( QualityTablesFormatter::RFIPercentageStatistic, singlePolStat, 0) << '\n'; } void WriteAntennae(casacore::MeasurementSet& ms, const std::vector& antennae) { casacore::MSAntenna antTable = ms.antenna(); casacore::ScalarColumn nameCol = casacore::ScalarColumn( antTable, antTable.columnName(casacore::MSAntennaEnums::NAME)); casacore::ScalarColumn stationCol = casacore::ScalarColumn( antTable, antTable.columnName(casacore::MSAntennaEnums::STATION)); casacore::ScalarColumn typeCol = casacore::ScalarColumn( antTable, antTable.columnName(casacore::MSAntennaEnums::TYPE)); casacore::ScalarColumn mountCol = casacore::ScalarColumn( antTable, antTable.columnName(casacore::MSAntennaEnums::MOUNT)); casacore::ArrayColumn positionCol = casacore::ArrayColumn( antTable, antTable.columnName(casacore::MSAntennaEnums::POSITION)); casacore::ScalarColumn dishDiameterCol = casacore::ScalarColumn( antTable, antTable.columnName(casacore::MSAntennaEnums::DISH_DIAMETER)); size_t rowIndex = antTable.nrow(); antTable.addRow(antennae.size()); for (std::vector::const_iterator antPtr = antennae.begin(); antPtr != antennae.end(); ++antPtr) { const AntennaInfo& ant = *antPtr; nameCol.put(rowIndex, ant.name); stationCol.put(rowIndex, ant.station); typeCol.put(rowIndex, ""); mountCol.put(rowIndex, ant.mount); casacore::Vector posArr(3); posArr[0] = ant.position.x; posArr[1] = ant.position.y; posArr[2] = ant.position.z; positionCol.put(rowIndex, posArr); dishDiameterCol.put(rowIndex, ant.diameter); ++rowIndex; } } void WritePolarizationForLinearPols(casacore::MeasurementSet& ms, bool flagRow = false) { casacore::MSPolarization polTable = ms.polarization(); casacore::ScalarColumn numCorrCol = casacore::ScalarColumn( polTable, polTable.columnName(casacore::MSPolarizationEnums::NUM_CORR)); casacore::ArrayColumn corrTypeCol = casacore::ArrayColumn( polTable, polTable.columnName(casacore::MSPolarizationEnums::CORR_TYPE)); casacore::ArrayColumn corrProductCol = casacore::ArrayColumn( polTable, polTable.columnName(casacore::MSPolarizationEnums::CORR_PRODUCT)); casacore::ScalarColumn flagRowCol = casacore::ScalarColumn( polTable, polTable.columnName(casacore::MSPolarizationEnums::FLAG_ROW)); const size_t rowIndex = polTable.nrow(); polTable.addRow(1); numCorrCol.put(rowIndex, 4); casacore::Vector cTypeVec(4); cTypeVec[0] = 9; cTypeVec[1] = 10; cTypeVec[2] = 11; cTypeVec[3] = 12; corrTypeCol.put(rowIndex, cTypeVec); casacore::Array cProdArr(casacore::IPosition(2, 2, 4)); casacore::Array::iterator i = cProdArr.begin(); *i = 0; ++i; *i = 0; ++i; *i = 0; ++i; *i = 1; ++i; *i = 1; ++i; *i = 0; ++i; *i = 1; ++i; *i = 1; corrProductCol.put(rowIndex, cProdArr); flagRowCol.put(rowIndex, flagRow); } void CombineStatistics(const std::string& result_filename, const std::vector& input_filenames) { if (!input_filenames.empty()) { const std::string& firstInFilename = *input_filenames.begin(); std::cout << "Combining " << input_filenames.size() << " sets into " << result_filename << '\n'; std::vector antennae; StatisticsCollection statisticsCollection; const HistogramCollection histogramCollection; std::cout << "Reading antenna table...\n"; const MSMetaData msMeta(firstInFilename); antennae.resize(msMeta.AntennaCount()); for (size_t i = 0; i != msMeta.AntennaCount(); ++i) antennae[i] = msMeta.GetAntennaInfo(i); for (std::vector::const_iterator i = input_filenames.begin(); i != input_filenames.end(); ++i) { std::cout << "Reading " << *i << "...\n"; // TODO read quality tables from all inFilenames QualityTablesFormatter formatter(*i); StatisticsCollection collectionPart; collectionPart.Load(formatter); if (i == input_filenames.begin()) statisticsCollection.SetPolarizationCount( collectionPart.PolarizationCount()); statisticsCollection.Add(collectionPart); } // Create main table casacore::TableDesc tableDesc = casacore::MS::requiredTableDesc(); const casacore::ArrayColumnDesc> dataColumnDesc = casacore::ArrayColumnDesc>( casacore::MS::columnName(casacore::MSMainEnums::DATA)); tableDesc.addColumn(dataColumnDesc); casacore::SetupNewTable newTab(result_filename, tableDesc, casacore::Table::New); casacore::MeasurementSet ms(newTab); ms.createDefaultSubtables(casacore::Table::New); std::cout << "Writing antenna table...\n"; WriteAntennae(ms, antennae); std::cout << "Writing polarization table (" << statisticsCollection.PolarizationCount() << " pols)...\n"; WritePolarizationForLinearPols(ms); std::cout << "Writing quality table...\n"; QualityTablesFormatter formatter(result_filename); statisticsCollection.Save(formatter); } } void RemoveStatistics(const std::string& filename) { QualityTablesFormatter formatter(filename); formatter.RemoveAllQualityTables(); } void PrintRfiSlope(const std::string& filename) { HistogramTablesFormatter histogramFormatter(filename); const unsigned polarizationCount = MSMetaData::PolarizationCount(filename); HistogramCollection collection(polarizationCount); collection.Load(histogramFormatter); const MSMetaData set(filename); std::cout << set.GetBandInfo(0).CenterFrequencyHz(); for (unsigned p = 0; p < polarizationCount; ++p) { LogHistogram histogram; collection.GetRFIHistogramForCrossCorrelations(p, histogram); std::cout << '\t' << histogram.NormalizedSlopeInRFIRegion(); } std::cout << '\n'; } void PrintRfiSlopePerBaseline(const std::string& filename, const char* dataColumnName) { HistogramTablesFormatter histogramFormatter(filename); const unsigned polarizationCount = MSMetaData::PolarizationCount(filename); HistogramCollection collection; CollectHistograms(filename, collection, 0, std::set(), dataColumnName); const MSMetaData set(filename); const size_t antennaCount = set.AntennaCount(); std::vector antennae(antennaCount); for (size_t a = 0; a < antennaCount; ++a) antennae[a] = set.GetAntennaInfo(a); HistogramCollection* summedCollection = collection.CreateSummedPolarizationCollection(); const std::map& histogramMap = summedCollection->GetRFIHistogram(0); PrintRFISlopeForHistogram(histogramMap, '*', &antennae[0]); delete summedCollection; for (unsigned p = 0; p < polarizationCount; ++p) { const std::map& histogramMap = collection.GetRFIHistogram(p); PrintRFISlopeForHistogram(histogramMap, '0' + p, &antennae[0]); } } void RemoveHistogram(const std::string& filename) { HistogramTablesFormatter histogramFormatter(filename); histogramFormatter.RemoveAll(); } } // namespace quality aoflagger-v3.5.1/quality/rayleighfitter.h0000664000175000017500000000266214752462134016660 0ustar oleole#ifndef RAYLEIGHFITTER_H #define RAYLEIGHFITTER_H #include "loghistogram.h" class RayleighFitter { public: RayleighFitter() : _fitLogarithmic(true), _hist(nullptr), _minVal(0.0), _maxVal(0.0) {} void Fit(double minVal, double maxVal, const LogHistogram& hist, double& sigma, double& n); static double SigmaEstimate(const LogHistogram& hist); static double SigmaEstimate(const LogHistogram& hist, double rangeStart, double rangeEnd); static double NEstimate(const LogHistogram& hist, double rangeStart, double rangeEnd); static void FindFitRangeUnderRFIContamination(double minPositiveAmplitude, double sigmaEstimate, double& minValue, double& maxValue); static double RayleighValue(double sigma, double n, double x) { double sigmaP2 = sigma * sigma; return n * x / (sigmaP2)*exp(-x * x / (2 * sigmaP2)); } static double ErrorOfFit(const LogHistogram& histogram, double rangeStart, double rangeEnd, double sigma, double n); bool FitLogarithmic() const { return _fitLogarithmic; } void SetFitLogarithmic(bool fitLogarithmic) { _fitLogarithmic = fitLogarithmic; } private: bool _fitLogarithmic; public: const LogHistogram* _hist; double _minVal, _maxVal; }; #endif aoflagger-v3.5.1/quality/statisticsderivator.h0000664000175000017500000003334214752462134017747 0ustar oleole#ifndef QUALITY__STATISTICS_DERIVATOR_H #define QUALITY__STATISTICS_DERIVATOR_H #include "statisticscollection.h" #include #include #include "../structures/antennainfo.h" #include "../structures/timefrequencydata.h" #include "../structures/timefrequencymetadata.h" struct stat; class StatisticsDerivator { public: explicit StatisticsDerivator(const StatisticsCollection& collection) : _collection(collection) {} std::complex GetComplexBaselineStatistic( QualityTablesFormatter::StatisticKind kind, unsigned antenna1, unsigned antenna2, unsigned polarization) const { const DefaultStatistics& statistics = _collection.BaselineStatistics().GetStatistics(antenna1, antenna2); return deriveComplex(kind, statistics, polarization); } std::complex GetComplexTimeStatistic( QualityTablesFormatter::StatisticKind kind, double time, unsigned polarization) const { const DefaultStatistics& statistics = _collection.TimeStatistics().find(time)->second; return deriveComplex(kind, statistics, polarization); } std::complex GetComplexFrequencyStatistic( QualityTablesFormatter::StatisticKind kind, double frequency, unsigned polarization) const { const DefaultStatistics& statistics = _collection.FrequencyStatistics().find(frequency)->second; return deriveComplex(kind, statistics, polarization); } static std::complex GetComplexStatistic( QualityTablesFormatter::StatisticKind kind, const DefaultStatistics& statistics, unsigned polarization) { return deriveComplex(kind, statistics, polarization); } static long double GetStatisticAmplitude( QualityTablesFormatter::StatisticKind kind, const DefaultStatistics& statistics, unsigned polarization) { const std::complex val = GetComplexStatistic(kind, statistics, polarization); return sqrtl(val.real() * val.real() + val.imag() * val.imag()); } static std::complex Variance(unsigned long n, std::complex sum, std::complex sumP2) { return deriveVariance(n, sum, sumP2); } static long double VarianceAmplitude(unsigned long n, std::complex sum, std::complex sumP2) { const std::complex variance = deriveVariance(n, sum, sumP2); return sqrtl(variance.real() * variance.real() + variance.imag() * variance.imag()); } static long double StandardDeviationAmplitude( unsigned long n, std::complex sum, std::complex sumP2) { const std::complex stdDev = deriveStandardDeviation(n, sum, sumP2); return sqrtl(stdDev.real() * stdDev.real() + stdDev.imag() * stdDev.imag()); } std::pair CreateTFData( QualityTablesFormatter::StatisticKind kind) { const std::map>& map = _collection.AllTimeStatistics(); std::set frequencies; std::set timesteps; // List the frequencies and timesteps for (std::map>::const_iterator i = map.begin(); i != map.end(); ++i) { const double frequency = i->first; frequencies.insert(frequency); const std::map& innerMap = i->second; for (std::map::const_iterator j = innerMap.begin(); j != innerMap.end(); ++j) { const double time = j->first; timesteps.insert(time); } } std::map freqIndices; std::map timeIndices; std::vector observationTimes; BandInfo band; size_t index = 0; for (std::set::const_iterator i = frequencies.begin(); i != frequencies.end(); ++i) { freqIndices.insert(std::pair(*i, index)); ChannelInfo channel; channel.frequencyIndex = index; channel.frequencyHz = *i; band.channels.push_back(channel); ++index; } index = 0; for (std::set::const_iterator i = timesteps.begin(); i != timesteps.end(); ++i) { timeIndices.insert(std::pair(*i, index)); observationTimes.push_back(*i); ++index; } // create the images const size_t pCount = _collection.PolarizationCount(); std::vector realImage(pCount), imagImage(pCount); Mask2DPtr mask = Mask2D::CreateSetMaskPtr(timesteps.size(), frequencies.size()); for (size_t p = 0; p < pCount; ++p) { realImage[p] = Image2D::CreateZeroImagePtr(timesteps.size(), frequencies.size()); imagImage[p] = Image2D::CreateZeroImagePtr(timesteps.size(), frequencies.size()); } // add the statistis for (std::map>::const_iterator i = map.begin(); i != map.end(); ++i) { const double frequency = i->first; const size_t freqIndex = freqIndices.find(frequency)->second; const std::map& innerMap = i->second; for (std::map::const_iterator j = innerMap.begin(); j != innerMap.end(); ++j) { const double time = j->first; const size_t timeIndex = timeIndices.find(time)->second; mask->SetValue(timeIndex, freqIndex, false); for (size_t p = 0; p < pCount; ++p) { const std::complex statValue = deriveComplex(kind, j->second, p); realImage[p]->SetValue(timeIndex, freqIndex, statValue.real()); imagImage[p]->SetValue(timeIndex, freqIndex, statValue.imag()); } } } TimeFrequencyData data = TimeFrequencyData::FromLinear( pCount, (Image2DCPtr*)&(realImage[0]), (Image2DCPtr*)&(imagImage[0])); data.SetGlobalMask(mask); TimeFrequencyMetaDataPtr metaData(new TimeFrequencyMetaData()); metaData->SetObservationTimes(observationTimes); metaData->SetBand(band); metaData->SetValueDescription(GetDescription(kind)); metaData->SetValueUnits(GetUnits(kind)); return std::pair(data, metaData); } static std::string GetDescription( QualityTablesFormatter::StatisticKind kind) { switch (kind) { case QualityTablesFormatter::MeanStatistic: return "Mean"; case QualityTablesFormatter::VarianceStatistic: return "Variance"; case QualityTablesFormatter::StandardDeviationStatistic: return "Standard deviation"; case QualityTablesFormatter::CountStatistic: return "Sample count"; case QualityTablesFormatter::DMeanStatistic: return "Differential mean"; case QualityTablesFormatter::DVarianceStatistic: return "Differential variance"; case QualityTablesFormatter::DStandardDeviationStatistic: return "Differential standard deviation"; case QualityTablesFormatter::DCountStatistic: return "Sample count in differential statistics"; case QualityTablesFormatter::RFICountStatistic: return "Sample count affected by RFI"; case QualityTablesFormatter::RFIRatioStatistic: return "RFI"; case QualityTablesFormatter::RFIPercentageStatistic: return "RFI"; case QualityTablesFormatter::SignalToNoiseStatistic: return "SNR"; default: return "Value"; } } static bool HasUnits(QualityTablesFormatter::StatisticKind kind) { return !GetUnits(kind).empty(); } static std::string GetUnits(QualityTablesFormatter::StatisticKind kind) { switch (kind) { case QualityTablesFormatter::MeanStatistic: case QualityTablesFormatter::StandardDeviationStatistic: case QualityTablesFormatter::DMeanStatistic: case QualityTablesFormatter::DVarianceStatistic: case QualityTablesFormatter::DStandardDeviationStatistic: return "Jy"; case QualityTablesFormatter::VarianceStatistic: return "Jy^2"; case QualityTablesFormatter::RFIPercentageStatistic: return "%"; case QualityTablesFormatter::CountStatistic: case QualityTablesFormatter::DCountStatistic: case QualityTablesFormatter::RFICountStatistic: return "count"; case QualityTablesFormatter::RFIRatioStatistic: return "ratio"; case QualityTablesFormatter::SignalToNoiseStatistic: default: return ""; } } static std::string GetDescWithUnits( QualityTablesFormatter::StatisticKind kind) { std::ostringstream str; str << GetDescription(kind); if (HasUnits(kind)) str << " (" << GetUnits(kind) << ")"; return str.str(); } private: template static std::complex deriveComplex( QualityTablesFormatter::StatisticKind kind, const DefaultStatistics& statistics, unsigned polarization) { switch (kind) { case QualityTablesFormatter::CountStatistic: return std::complex(statistics.count[polarization], 0.0); break; case QualityTablesFormatter::MeanStatistic: return statistics.Mean(polarization); break; case QualityTablesFormatter::SumStatistic: return statistics.Sum(polarization); break; case QualityTablesFormatter::SumP2Statistic: return statistics.SumP2(polarization); break; case QualityTablesFormatter::VarianceStatistic: return deriveVariance(statistics.count[polarization], statistics.sum[polarization], statistics.sumP2[polarization]); break; case QualityTablesFormatter::StandardDeviationStatistic: return deriveStandardDeviation(statistics.count[polarization], statistics.sum[polarization], statistics.sumP2[polarization]); break; case QualityTablesFormatter::DCountStatistic: return std::complex(statistics.dCount[polarization], 0.0f); break; case QualityTablesFormatter::DMeanStatistic: return statistics.DMean(polarization); break; case QualityTablesFormatter::DSumStatistic: return statistics.DSum(polarization); break; case QualityTablesFormatter::DSumP2Statistic: return statistics.DSumP2(polarization); break; case QualityTablesFormatter::DVarianceStatistic: return deriveVariance(statistics.dCount[polarization], statistics.dSum[polarization], statistics.dSumP2[polarization]); break; case QualityTablesFormatter::DStandardDeviationStatistic: return deriveStandardDeviation(statistics.dCount[polarization], statistics.dSum[polarization], statistics.dSumP2[polarization]); break; case QualityTablesFormatter::RFIRatioStatistic: return std::complex((double)statistics.rfiCount[polarization] / (statistics.count[polarization] + statistics.rfiCount[polarization]), 0.0f); break; case QualityTablesFormatter::RFIPercentageStatistic: return std::complex(100.0 * (double)statistics.rfiCount[polarization] / (statistics.count[polarization] + statistics.rfiCount[polarization]), 0.0f); break; case QualityTablesFormatter::RFICountStatistic: return std::complex(statistics.rfiCount[polarization], 0.0f); break; case QualityTablesFormatter::SignalToNoiseStatistic: { const std::complex stddev = deriveComplex( QualityTablesFormatter::DStandardDeviationStatistic, statistics, polarization); return std::complex( fabsl(statistics.Mean(polarization).real() / stddev.real()), fabsl(statistics.Mean(polarization).imag() / stddev.imag())); } default: throw std::runtime_error("Can not derive requested statistic"); } } template static std::complex deriveVariance(unsigned long n, std::complex sum, std::complex sumP2) { return std::complex(deriveVarianceSingle(n, sum.real(), sumP2.real()), deriveVarianceSingle(n, sum.imag(), sumP2.imag())); } template static std::complex deriveStandardDeviation( unsigned long n, std::complex sum, std::complex sumP2) { return std::complex( deriveStandardDeviationSingle(n, sum.real(), sumP2.real()), deriveStandardDeviationSingle(n, sum.imag(), sumP2.imag())); } template static T deriveVarianceSingle(unsigned long n, T sum, T sumP2) { T sumMeanSquared = sum * sum / n; return (sumP2 - sumMeanSquared) / (n - 1.0); } template static T deriveStandardDeviationSingle(unsigned long n, T sum, T sumP2) { T sumMeanSquared = sum * sum / n; return sqrt((sumP2 - sumMeanSquared) / n); } const StatisticsCollection& _collection; }; #endif aoflagger-v3.5.1/quality/collector.cpp0000664000175000017500000002041314752462134016151 0ustar oleole#include "collector.h" #include "../structures/msmetadata.h" #include "../util/progress/progresslistener.h" #include "histogramcollection.h" #include "statisticscollection.h" #include #include #include #include Collector::Collector() : _mode(CollectDefault), _dataColumnName("DATA"), _intervalStart(0), _intervalEnd(0), _flaggedTimesteps(0) {} Collector::~Collector() = default; void Collector::Collect(const std::string& filename, StatisticsCollection& statisticsCollection, HistogramCollection& histogramCollection, ProgressListener& progressListener) { progressListener.OnStartTask("Collecting statistics"); std::unique_ptr ms(new MSMetaData(filename)); const size_t polarizationCount = ms->PolarizationCount(), bandCount = ms->BandCount(); const bool ignoreChannelZero = ms->IsChannelZeroRubish() && _mode != CollectTimeFrequency; const std::string stationName = ms->GetStationName(); std::vector bands(bandCount); std::vector> frequencies(bandCount); for (unsigned b = 0; b < bandCount; ++b) { bands[b] = ms->GetBandInfo(b); frequencies[b].resize(bands[b].channels.size()); for (unsigned c = 0; c < bands[b].channels.size(); ++c) { frequencies[b][c] = bands[b].channels[c].frequencyHz; } } ms.reset(); // Initialize statisticscollection _statisticsCollection = &statisticsCollection; _histogramCollection = &histogramCollection; statisticsCollection.SetPolarizationCount(polarizationCount); const size_t nCPU = aocommon::system::ProcessorCount(); _threadStats.clear(); for (size_t i = 0; i != nCPU; ++i) _threadStats.emplace_back(polarizationCount); if (_mode != CollectHistograms) { for (unsigned b = 0; b < bandCount; ++b) { if (ignoreChannelZero) statisticsCollection.InitializeBand(b, (frequencies[b].data() + 1), bands[b].channels.size() - 1); else statisticsCollection.InitializeBand(b, frequencies[b].data(), bands[b].channels.size()); for (size_t i = 0; i != nCPU; ++i) { if (ignoreChannelZero) _threadStats[i].InitializeBand(b, (frequencies[b].data() + 1), bands[b].channels.size() - 1); else _threadStats[i].InitializeBand(b, frequencies[b].data(), bands[b].channels.size()); } } } // Initialize Histograms collection histogramCollection.SetPolarizationCount(polarizationCount); // get columns const casacore::Table table(filename); const casacore::ArrayColumn dataColumn(table, _dataColumnName); const casacore::ArrayColumn flagColumn(table, "FLAG"); const casacore::ScalarColumn timeColumn(table, "TIME"); casacore::ScalarColumn antenna1Column(table, "ANTENNA1"), antenna2Column(table, "ANTENNA2"), windowColumn(table, "DATA_DESC_ID"); const size_t channelCount = bands[0].channels.size(); _correlatorFlags.assign(channelCount, false); _correlatorFlagsForBadAntenna.assign(channelCount, true); aocommon::Lane workLane(nCPU * 4); std::vector threads; for (size_t i = 0; i != nCPU; ++i) threads.emplace_back( [&, i]() { process(&workLane, &_threadStats[i], polarizationCount); }); const bool hasInterval = !(_intervalStart == 0 && _intervalEnd == 0); const size_t nrow = table.nrow(); size_t timestepIndex = (size_t)-1; double prevtime = -1.0; for (size_t row = 0; row != nrow; ++row) { Work work; work.time = timeColumn(row); if (work.time != prevtime) { ++timestepIndex; prevtime = work.time; } if (!hasInterval || (timestepIndex >= _intervalStart && timestepIndex < _intervalEnd)) { work.antenna1Index = antenna1Column(row), work.antenna2Index = antenna2Column(row), work.bandIndex = windowColumn(row); work.timestepIndex = timestepIndex; const BandInfo& band = bands[work.bandIndex]; const casacore::Array dataArray = dataColumn(row); const casacore::Array flagArray = flagColumn(row); work.hasFlaggedAntenna = _flaggedAntennae.find(work.antenna1Index) != _flaggedAntennae.end() || _flaggedAntennae.find(work.antenna2Index) != _flaggedAntennae.end(); casacore::Array::const_contiter dataIter = dataArray.cbegin(); casacore::Array::const_contiter flagIter = flagArray.cbegin(); const unsigned startChannel = ignoreChannelZero ? 1 : 0; if (ignoreChannelZero) { dataIter += polarizationCount; flagIter += polarizationCount; } work.nChan = band.channels.size() - startChannel; work.isRFI.resize(polarizationCount); work.samples.resize(polarizationCount); for (unsigned p = 0; p < polarizationCount; ++p) { work.isRFI[p].resize(band.channels.size()); work.samples[p].resize(band.channels.size()); } for (unsigned channel = startChannel; channel < band.channels.size(); ++channel) { for (unsigned p = 0; p < polarizationCount; ++p) { work.samples[p][channel - startChannel] = *dataIter; work.isRFI[p][channel - startChannel] = *flagIter; ++dataIter; ++flagIter; } } workLane.write(std::move(work)); } progressListener.OnProgress(row, nrow); } workLane.write_end(); for (size_t i = 0; i != nCPU; ++i) threads[i].join(); progressListener.OnFinish(); } void Collector::process(aocommon::Lane* workLane, StatisticsCollection* stats, size_t polarizationCount) { Work work; HistogramCollection histos(polarizationCount); while (workLane->read(work)) { for (unsigned p = 0; p < polarizationCount; ++p) { switch (_mode) { case CollectDefault: if (work.hasFlaggedAntenna || work.timestepIndex < _flaggedTimesteps) stats->Add(work.antenna1Index, work.antenna2Index, work.time, work.bandIndex, p, &reinterpret_cast(work.samples[p].data())[0], &reinterpret_cast(work.samples[p].data())[1], work.isRFI[p].data(), _correlatorFlagsForBadAntenna.data(), work.nChan, 2, 1, 1); else stats->Add(work.antenna1Index, work.antenna2Index, work.time, work.bandIndex, p, &reinterpret_cast(work.samples[p].data())[0], &reinterpret_cast(work.samples[p].data())[1], work.isRFI[p].data(), _correlatorFlags.data(), work.nChan, 2, 1, 1); break; case CollectHistograms: histos.Add(work.antenna1Index, work.antenna2Index, p, work.samples[p].data(), work.isRFI[p].data(), work.nChan); break; case CollectTimeFrequency: if (work.hasFlaggedAntenna || work.timestepIndex < _flaggedTimesteps) stats->Add(work.antenna1Index, work.antenna2Index, work.time, work.bandIndex, p, &reinterpret_cast(work.samples[p].data())[0], &reinterpret_cast(work.samples[p].data())[1], work.isRFI[p].data(), _correlatorFlagsForBadAntenna.data(), work.nChan, 2, 1, 1); else stats->AddToTimeFrequency( work.antenna1Index, work.antenna2Index, work.time, work.bandIndex, p, &reinterpret_cast(work.samples[p].data())[0], &reinterpret_cast(work.samples[p].data())[1], work.isRFI[p].data(), _correlatorFlags.data(), work.nChan, 2, 1, 1); break; } } } const std::unique_lock lock(_mutex); _statisticsCollection->Add(*stats); _histogramCollection->Add(histos); } aoflagger-v3.5.1/quality/combine.cpp0000664000175000017500000000273014752462134015601 0ustar oleole#include "combine.h" #include "../structures/msmetadata.h" namespace quality { FileContents ReadAndCombine(const std::vector& files, bool verbose) { FileContents result; size_t n_polarizations = 0; for (size_t i = 0; i != files.size(); ++i) { const std::string& filename = files[i]; if (n_polarizations == 0) { n_polarizations = MSMetaData::PolarizationCount(filename); result.statistics_collection.SetPolarizationCount(n_polarizations); result.histogram_collection.SetPolarizationCount(n_polarizations); } else { if (n_polarizations != MSMetaData::PolarizationCount(filename)) { throw std::runtime_error( "Can't combine measurement set quality statistics with different " "number of polarizations"); } } if (verbose) std::cout << " (" << (i + 1) << "/" << files.size() << ") Adding " << filename << " to statistics...\n"; QualityTablesFormatter qualityTables(filename); StatisticsCollection statCollection(n_polarizations); statCollection.Load(qualityTables); result.statistics_collection.Add(statCollection); HistogramTablesFormatter histogramTables(filename); if (histogramTables.HistogramsExist()) { HistogramCollection histCollection(n_polarizations); histCollection.Load(histogramTables); result.histogram_collection.Add(histCollection); } } return result; } } // namespace quality aoflagger-v3.5.1/quality/histogramtablesformatter.h0000664000175000017500000001173514752462134020753 0ustar oleole#ifndef HISTOGRAM_TABLES_FORMATTER_H #define HISTOGRAM_TABLES_FORMATTER_H #include #include #include #include class HistogramTablesFormatter { public: enum TableKind { HistogramCountTable, HistogramTypeTable }; struct HistogramItem { double binStart; double binEnd; double count; }; enum HistogramType { TotalHistogram, RFIHistogram }; explicit HistogramTablesFormatter(const std::string& measurementSetName) : _measurementSet(nullptr), _measurementSetName(measurementSetName), _typeTable(nullptr), _countTable(nullptr) {} ~HistogramTablesFormatter() { Close(); } void Close() { _countTable.reset(); _typeTable.reset(); closeMainTable(); } std::string CountTableName() const { return "QUALITY_HISTOGRAM_COUNT"; } std::string TypeTableName() const { return "QUALITY_HISTOGRAM_TYPE"; } std::string TableName(enum TableKind table) const { switch (table) { case HistogramCountTable: return CountTableName(); case HistogramTypeTable: return TypeTableName(); default: return ""; } } std::string TypeToName(HistogramType type) const { switch (type) { case TotalHistogram: return "Total"; case RFIHistogram: return "RFI"; default: return std::string(); } } std::string TableFilename(enum TableKind table) const { return _measurementSetName + '/' + TableName(table); } bool TableExists(enum TableKind table) const { return _measurementSet->isReadable(TableFilename(table)); } bool IsAvailable(unsigned typeIndex) { if (!TableExists(HistogramCountTable) || !TableExists(HistogramTypeTable)) return false; return hasOneEntry(typeIndex); } void InitializeEmptyTables(); void RemoveTable(enum TableKind table); void StoreValue(unsigned typeIndex, double binStart, double binEnd, double count); void QueryHistogram(unsigned typeIndex, std::vector& histogram); unsigned QueryTypeIndex(enum HistogramType type, unsigned polarizationIndex); bool QueryTypeIndex(enum HistogramType type, unsigned polarizationIndex, unsigned& destTypeIndex); unsigned StoreOrQueryTypeIndex(enum HistogramType type, unsigned polarizationIndex) { unsigned typeIndex; if (QueryTypeIndex(type, polarizationIndex, typeIndex)) return typeIndex; else return StoreType(type, polarizationIndex); } unsigned StoreType(enum HistogramType type, unsigned polarizationIndex); bool HistogramsExist() { return TableExists(HistogramCountTable) && TableExists(HistogramTypeTable); } void RemoveAll() { RemoveTable(HistogramCountTable); RemoveTable(HistogramTypeTable); } private: HistogramTablesFormatter(const HistogramTablesFormatter&) = delete; // don't allow copies void operator=(const HistogramTablesFormatter&) = delete; // don't allow assignment static const std::string ColumnNameType; static const std::string ColumnNameName; static const std::string ColumnNamePolarization; static const std::string ColumnNameBinStart; static const std::string ColumnNameBinEnd; static const std::string ColumnNameCount; std::unique_ptr _measurementSet; const std::string _measurementSetName; std::unique_ptr _typeTable; std::unique_ptr _countTable; bool hasOneEntry(unsigned typeIndex); void removeTypeEntry(enum HistogramType type, unsigned polarizationIndex); void removeEntries(enum TableKind table); void addTimeColumn(casacore::TableDesc& tableDesc); void addFrequencyColumn(casacore::TableDesc& tableDesc); void addValueColumn(casacore::TableDesc& tableDesc); void createTable(enum TableKind table) { switch (table) { case HistogramTypeTable: createTypeTable(); break; case HistogramCountTable: createCountTable(); break; default: break; } } void createTypeTable(); void createCountTable(); unsigned findFreeTypeIndex(casacore::Table& typeTable); void openMainTable(bool needWrite); void closeMainTable() { _measurementSet.reset(); } void openTable(TableKind table, bool needWrite, std::unique_ptr& tablePtr); void openTypeTable(bool needWrite) { openTable(HistogramTypeTable, needWrite, _typeTable); } void openCountTable(bool needWrite) { openTable(HistogramCountTable, needWrite, _countTable); } casacore::Table& getTable(TableKind table, bool needWrite) { std::unique_ptr* tablePtr = nullptr; switch (table) { case HistogramTypeTable: tablePtr = &_typeTable; break; case HistogramCountTable: tablePtr = &_countTable; break; } assert(tablePtr); openTable(table, needWrite, *tablePtr); return **tablePtr; } }; #endif aoflagger-v3.5.1/quality/histogramtablesformatter.cpp0000664000175000017500000002331615106657110021276 0ustar oleole#include "histogramtablesformatter.h" #include #include #include #include #include #include #include #include #include "statisticalvalue.h" const std::string HistogramTablesFormatter::ColumnNameType = "TYPE"; const std::string HistogramTablesFormatter::ColumnNameName = "NAME"; const std::string HistogramTablesFormatter::ColumnNameCount = "COUNT"; const std::string HistogramTablesFormatter::ColumnNamePolarization = "POLARIZATION"; const std::string HistogramTablesFormatter::ColumnNameBinStart = "BIN_START"; const std::string HistogramTablesFormatter::ColumnNameBinEnd = "BIN_END"; void HistogramTablesFormatter::InitializeEmptyTables() { if (TableExists(HistogramCountTable)) removeEntries(HistogramCountTable); else createCountTable(); if (TableExists(HistogramTypeTable)) removeEntries(HistogramTypeTable); else createTypeTable(); } void HistogramTablesFormatter::RemoveTable(enum TableKind table) { if (TableExists(table)) { Close(); openMainTable(true); if (_measurementSet->keywordSet().isDefined(TableName(table))) _measurementSet->rwKeywordSet().removeField(TableName(table)); if (_measurementSet->isReadable(TableFilename(table))) casacore::TableUtil::deleteTable(TableFilename(table)); } } unsigned HistogramTablesFormatter::QueryTypeIndex(enum HistogramType type, unsigned polarizationIndex) { unsigned kindIndex; if (!QueryTypeIndex(type, polarizationIndex, kindIndex)) throw std::runtime_error( "getKindIndex(): Requested statistic kind not available."); return kindIndex; } bool HistogramTablesFormatter::QueryTypeIndex(enum HistogramType type, unsigned polarizationIndex, unsigned& destTypeIndex) { openTypeTable(false); const casacore::ROScalarColumn typeColumn(*_typeTable, ColumnNameType); const casacore::ROScalarColumn polarizationColumn( *_typeTable, ColumnNamePolarization); const casacore::ROScalarColumn nameColumn(*_typeTable, ColumnNameName); const casacore::String nameToFind(TypeToName(type)); const unsigned nrRow = _typeTable->nrow(); for (unsigned i = 0; i < nrRow; ++i) { if ((unsigned)polarizationColumn(i) == polarizationIndex && nameColumn(i) == nameToFind) { destTypeIndex = typeColumn(i); return true; } } return false; } bool HistogramTablesFormatter::hasOneEntry(unsigned typeIndex) { const casacore::Table& casaTable = getTable(HistogramCountTable, false); const casacore::ROScalarColumn typeColumn(casaTable, ColumnNameType); const unsigned nrRow = casaTable.nrow(); for (unsigned i = 0; i < nrRow; ++i) { if (typeColumn(i) == (int)typeIndex) return true; } return false; } void HistogramTablesFormatter::createTypeTable() { casacore::TableDesc tableDesc(TypeTableName() + "_TYPE", "1.0", casacore::TableDesc::Scratch); tableDesc.comment() = "Couples the TYPE column in the QUALITY_HISTOGRAM_COUNT table to the " "name and polarization of the histogram"; tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameType, "Index of the statistic kind")); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNamePolarization, "Index of the polarization corresponding to the main table")); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameName, "Name of the statistic")); casacore::SetupNewTable newTableSetup(TableFilename(HistogramTypeTable), tableDesc, casacore::Table::New); const casacore::Table newTable(newTableSetup); openMainTable(true); _measurementSet->rwKeywordSet().defineTable(TypeTableName(), newTable); } void HistogramTablesFormatter::createCountTable() { casacore::TableDesc tableDesc(CountTableName() + "_TYPE", "1.0", casacore::TableDesc::Scratch); tableDesc.comment() = "Histograms of the data in the main table"; tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameType, "Index of the statistic kind")); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameBinStart, "Lower start value of the bin corresponding to the count")); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameBinEnd, "Higher end value of the bin corresponding to the count")); tableDesc.addColumn( casacore::ScalarColumnDesc(ColumnNameCount, "Histogram y-value")); casacore::SetupNewTable newTableSetup(TableFilename(HistogramCountTable), tableDesc, casacore::Table::New); const casacore::Table newTable(newTableSetup); openMainTable(true); _measurementSet->rwKeywordSet().defineTable(CountTableName(), newTable); } unsigned HistogramTablesFormatter::StoreType(enum HistogramType type, unsigned polarizationIndex) { openTypeTable(true); const unsigned typeIndex = findFreeTypeIndex(*_typeTable); const unsigned newRow = _typeTable->nrow(); _typeTable->addRow(); casacore::ScalarColumn typeColumn(*_typeTable, ColumnNameType); casacore::ScalarColumn polarizationColumn(*_typeTable, ColumnNamePolarization); casacore::ScalarColumn nameColumn(*_typeTable, ColumnNameName); typeColumn.put(newRow, typeIndex); polarizationColumn.put(newRow, polarizationIndex); nameColumn.put(newRow, TypeToName(type)); return typeIndex; } unsigned HistogramTablesFormatter::findFreeTypeIndex( casacore::Table& typeTable) { int maxIndex = 0; const casacore::ROScalarColumn typeColumn(typeTable, ColumnNameType); const unsigned nrRow = typeTable.nrow(); for (unsigned i = 0; i < nrRow; ++i) { if (typeColumn(i) > maxIndex) maxIndex = typeColumn(i); } return maxIndex + 1; } void HistogramTablesFormatter::openTable( enum TableKind table, bool needWrite, std::unique_ptr& tablePtr) { if (!tablePtr) { openMainTable(false); tablePtr = std::make_unique( _measurementSet->keywordSet().asTable(TableName(table))); if (needWrite) tablePtr->reopenRW(); } else { if (needWrite && !tablePtr->isWritable()) tablePtr->reopenRW(); } } void HistogramTablesFormatter::StoreValue(unsigned typeIndex, double binStart, double binEnd, double count) { openCountTable(true); const unsigned newRow = _countTable->nrow(); _countTable->addRow(); casacore::ScalarColumn typeColumn(*_countTable, ColumnNameType); casacore::ScalarColumn binStartColumn(*_countTable, ColumnNameBinStart); casacore::ScalarColumn binEndColumn(*_countTable, ColumnNameBinEnd); casacore::ScalarColumn countColumn(*_countTable, ColumnNameCount); typeColumn.put(newRow, typeIndex); binStartColumn.put(newRow, binStart); binEndColumn.put(newRow, binEnd); countColumn.put(newRow, count); } void HistogramTablesFormatter::removeTypeEntry(enum HistogramType type, unsigned polarizationIndex) { openTypeTable(true); const casacore::ScalarColumn polarizationColumn(*_typeTable, ColumnNamePolarization); const casacore::ScalarColumn nameColumn(*_typeTable, ColumnNameName); const unsigned nrRow = _typeTable->nrow(); const casacore::String typeName(TypeToName(type)); for (unsigned i = 0; i < nrRow; ++i) { if (nameColumn(i) == typeName && (unsigned)polarizationColumn(i) == polarizationIndex) { _typeTable->removeRow(i); break; } } } void HistogramTablesFormatter::removeEntries(enum TableKind table) { casacore::Table& casaTable = getTable(table, true); const unsigned nrRow = casaTable.nrow(); for (int i = nrRow - 1; i >= 0; --i) { casaTable.removeRow(i); } } void HistogramTablesFormatter::QueryHistogram( unsigned typeIndex, std::vector& histogram) { const casacore::Table& table(getTable(HistogramCountTable, false)); const unsigned nrRow = table.nrow(); const casacore::ROScalarColumn typeColumn(table, ColumnNameType); const casacore::ROScalarColumn binStartColumn(table, ColumnNameBinStart); const casacore::ROScalarColumn binEndColumn(table, ColumnNameBinEnd); const casacore::ROScalarColumn countColumn(table, ColumnNameCount); for (unsigned i = 0; i < nrRow; ++i) { if (typeColumn(i) == (int)typeIndex) { HistogramItem item; item.binStart = binStartColumn(i); item.binEnd = binEndColumn(i); item.count = countColumn(i); histogram.push_back(item); } } } void HistogramTablesFormatter::openMainTable(bool needWrite) { if (_measurementSet == nullptr) { if (needWrite) _measurementSet = std::make_unique( _measurementSetName, casacore::Table::Update); else _measurementSet = std::make_unique(_measurementSetName); } else if (needWrite) { if (!_measurementSet->isWritable()) _measurementSet->reopenRW(); } } aoflagger-v3.5.1/quality/qualitytablesformatter.cpp0000664000175000017500000006327715106657110021003 0ustar oleole#include "qualitytablesformatter.h" #include #include #include #include #include #include #include #include #include #include #include "statisticalvalue.h" const std::string QualityTablesFormatter::_kindToNameTable[] = { "Count", "Sum", "Mean", "RFICount", "RFISum", "RFIMean", "RFIRatio", "RFIPercentage", "FlaggedCount", "FlaggedRatio", "SumP2", "SumP3", "SumP4", "Variance", "VarianceOfVariance", "StandardDeviation", "Skewness", "Kurtosis", "SignalToNoise", "DSum", "DMean", "DSumP2", "DSumP3", "DSumP4", "DVariance", "DVarianceOfVariance", "DStandardDeviation", "DCount", "BadSolutionCount", "CorrectCount", "CorrectedMean", "CorrectedSumP2", "CorrectedDCount", "CorrectedDMean", "CorrectedDSumP2", "FTSum", "FTSumP2"}; const std::string QualityTablesFormatter::_tableToNameTable[] = { "QUALITY_KIND_NAME", "QUALITY_TIME_STATISTIC", "QUALITY_FREQUENCY_STATISTIC", "QUALITY_BASELINE_STATISTIC", "QUALITY_BASELINE_TIME_STATISTIC"}; const enum QualityTablesFormatter::QualityTable QualityTablesFormatter::_dimensionToTableTable[] = { TimeStatisticTable, FrequencyStatisticTable, BaselineStatisticTable, BaselineTimeStatisticTable}; const std::string QualityTablesFormatter::ColumnNameAntenna1 = "ANTENNA1"; const std::string QualityTablesFormatter::ColumnNameAntenna2 = "ANTENNA2"; const std::string QualityTablesFormatter::ColumnNameFrequency = "FREQUENCY"; const std::string QualityTablesFormatter::ColumnNameKind = "KIND"; const std::string QualityTablesFormatter::ColumnNameName = "NAME"; const std::string QualityTablesFormatter::ColumnNameTime = "TIME"; const std::string QualityTablesFormatter::ColumnNameValue = "VALUE"; enum QualityTablesFormatter::StatisticKind QualityTablesFormatter::NameToKind( const std::string& kindName) { for (unsigned i = 0; i < 37; ++i) { if (kindName == _kindToNameTable[i]) return (QualityTablesFormatter::StatisticKind)i; } throw std::runtime_error("Statistics kind not known"); } unsigned QualityTablesFormatter::QueryKindIndex(enum StatisticKind kind) { unsigned kindIndex; if (!QueryKindIndex(kind, kindIndex)) throw std::runtime_error( "getKindIndex(): Requested statistic kind not available."); return kindIndex; } bool QualityTablesFormatter::QueryKindIndex(enum StatisticKind kind, unsigned& destKindIndex) { openKindNameTable(false); const casacore::ROScalarColumn kindColumn(*_kindNameTable, ColumnNameKind); const casacore::ROScalarColumn nameColumn(*_kindNameTable, ColumnNameName); const casacore::String nameToFind(KindToName(kind)); const unsigned nrRow = _kindNameTable->nrow(); for (unsigned i = 0; i < nrRow; ++i) { if (nameColumn(i) == nameToFind) { destKindIndex = kindColumn(i); return true; } } return false; } bool QualityTablesFormatter::IsStatisticAvailable( enum StatisticDimension dimension, enum StatisticKind kind) { const QualityTable table = DimensionToTable(dimension); if (!TableExists(KindNameTable) || !TableExists(table)) return false; unsigned kindIndex; if (!QueryKindIndex(kind, kindIndex)) return false; return hasOneEntry(table, kindIndex); } void QualityTablesFormatter::InitializeEmptyStatistic( enum StatisticDimension dimension, enum StatisticKind kind, unsigned polarizationCount) { if (!TableExists(KindNameTable)) createKindNameTable(); const QualityTable table = DimensionToTable(dimension); if (!TableExists(table)) { InitializeEmptyTable(table, polarizationCount); } else { removeStatisticFromStatTable(table, kind); } } void QualityTablesFormatter::InitializeEmptyTable(enum QualityTable table, unsigned polarizationCount) { if (TableExists(table)) removeEntries(table); else createTable(table, polarizationCount); } void QualityTablesFormatter::RemoveTable(enum QualityTable table) { if (TableExists(table)) { Close(); openMainTable(true); if (_measurementSet->keywordSet().isDefined(TableToName(table))) _measurementSet->rwKeywordSet().removeField(TableToName(table)); if (_measurementSet->isReadable(TableToFilename(table))) casacore::TableUtil::deleteTable(TableToFilename(table)); } } bool QualityTablesFormatter::hasOneEntry(enum QualityTable table, unsigned kindIndex) { const casacore::Table& casaTable = getTable(table, false); const casacore::ROScalarColumn kindColumn(casaTable, ColumnNameKind); const unsigned nrRow = casaTable.nrow(); for (unsigned i = 0; i < nrRow; ++i) { if (kindColumn(i) == (int)kindIndex) return true; } return false; } /** * Will add an empty table to the measurement set named "QUALITY_KIND_NAME" and * initialize its default column. * This table can hold a list of quality statistic types that are referred to in * the statistic value tables. */ void QualityTablesFormatter::createKindNameTable() { casacore::TableDesc tableDesc("QUALITY_KIND_NAME_TYPE", QUALITY_TABLES_VERSION_STR, casacore::TableDesc::Scratch); tableDesc.comment() = "Couples the KIND column in the other quality tables to the name of a " "statistic (e.g. Mean)"; tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameKind, "Index of the statistic kind")); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameName, "Name of the statistic")); casacore::SetupNewTable newTableSetup(TableToFilename(KindNameTable), tableDesc, casacore::Table::New); const casacore::Table newTable(newTableSetup); openMainTable(true); _measurementSet->rwKeywordSet().defineTable(TableToName(KindNameTable), newTable); } /** * Add the time column to the table descriptor. Used by create..Table() methods. * It holds "Measure"s of time, which is what casacore defines as a value * including a unit and a reference frame. */ void QualityTablesFormatter::addTimeColumn(casacore::TableDesc& tableDesc) { const casacore::ScalarColumnDesc timeDesc( ColumnNameTime, "Central time of statistic"); tableDesc.addColumn(timeDesc); const casacore::TableMeasRefDesc measRef(casacore::MEpoch::UTC); const casacore::TableMeasValueDesc measVal(tableDesc, ColumnNameTime); casacore::TableMeasDesc mepochCol(measVal, measRef); mepochCol.write(tableDesc); } /** * Add the frequency column to the table descriptor. Used by create..Table() * methods. It holds "Quantum"s of frequency, which is what casacore defines as * a value including a unit (Hertz). */ void QualityTablesFormatter::addFrequencyColumn( casacore::TableDesc& tableDesc) { const casacore::ScalarColumnDesc freqDesc( ColumnNameFrequency, "Central frequency of statistic bin"); tableDesc.addColumn(freqDesc); const casacore::Unit hertzUnit("Hz"); casacore::TableQuantumDesc quantDesc(tableDesc, ColumnNameFrequency, hertzUnit); quantDesc.write(tableDesc); } /** * Add value column to the table descriptor. Used by create..Table() methods. * It consist of an array of statistics, each element holds a polarization. */ void QualityTablesFormatter::addValueColumn(casacore::TableDesc& tableDesc, unsigned polarizationCount) { casacore::IPosition shape(1); shape[0] = polarizationCount; const casacore::ArrayColumnDesc valDesc( ColumnNameValue, "Value of statistic", shape, casacore::ColumnDesc::Direct); tableDesc.addColumn(valDesc); } /** * Will add an empty table to the measurement set named "QUALITY_TIME_STATISTIC" * and initialize its default column. This table can hold several statistic * kinds per time step. * @param polarizationCount specifies the nr polarizations. This is required for * the shape of the value column. */ void QualityTablesFormatter::createTimeStatisticTable( unsigned polarizationCount) { casacore::TableDesc tableDesc("QUALITY_TIME_STATISTIC_TYPE", QUALITY_TABLES_VERSION_STR, casacore::TableDesc::Scratch); tableDesc.comment() = "Statistics over time"; addTimeColumn(tableDesc); addFrequencyColumn(tableDesc); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameKind, "Index of the statistic kind")); addValueColumn(tableDesc, polarizationCount); casacore::SetupNewTable newTableSetup(TableToFilename(TimeStatisticTable), tableDesc, casacore::Table::New); const casacore::Table newTable(newTableSetup); openMainTable(true); _measurementSet->rwKeywordSet().defineTable(TableToName(TimeStatisticTable), newTable); } /** * Will add an empty table to the measurement set named * "QUALITY_FREQUENCY_STATISTIC" and initialize its default column. This table * can hold several statistic kinds per time step. * @param polarizationCount specifies the nr polarizations. This is required for * the shape of the value column. */ void QualityTablesFormatter::createFrequencyStatisticTable( unsigned polarizationCount) { casacore::TableDesc tableDesc("QUALITY_FREQUENCY_STATISTIC_TYPE", QUALITY_TABLES_VERSION_STR, casacore::TableDesc::Scratch); tableDesc.comment() = "Statistics over frequency"; addFrequencyColumn(tableDesc); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameKind, "Index of the statistic kind")); addValueColumn(tableDesc, polarizationCount); casacore::SetupNewTable newTableSetup( TableToFilename(FrequencyStatisticTable), tableDesc, casacore::Table::New); const casacore::Table newTable(newTableSetup); openMainTable(true); _measurementSet->rwKeywordSet().defineTable( TableToName(FrequencyStatisticTable), newTable); } /** * Will add an empty table to the measurement set named * "QUALITY_BASELINE_STATISTIC" and initialize its default column. This table * can hold several statistic kinds per time step. * @param polarizationCount specifies the nr polarizations. This is required for * the shape of the value column. */ void QualityTablesFormatter::createBaselineStatisticTable( unsigned polarizationCount) { casacore::TableDesc tableDesc("QUALITY_BASELINE_STATISTIC_TYPE", QUALITY_TABLES_VERSION_STR, casacore::TableDesc::Scratch); tableDesc.comment() = "Statistics per baseline"; tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameAntenna1, "Index of first antenna")); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameAntenna2, "Index of second antenna")); addFrequencyColumn(tableDesc); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameKind, "Index of the statistic kind")); addValueColumn(tableDesc, polarizationCount); casacore::SetupNewTable newTableSetup(TableToFilename(BaselineStatisticTable), tableDesc, casacore::Table::New); const casacore::Table newTable(newTableSetup); openMainTable(true); _measurementSet->rwKeywordSet().defineTable( TableToName(BaselineStatisticTable), newTable); } void QualityTablesFormatter::createBaselineTimeStatisticTable( unsigned polarizationCount) { casacore::TableDesc tableDesc("QUALITY_BASELINE_TIME_STATISTIC_TYPE", QUALITY_TABLES_VERSION_STR, casacore::TableDesc::Scratch); tableDesc.comment() = "Statistics per baseline"; addTimeColumn(tableDesc); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameAntenna1, "Index of first antenna")); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameAntenna2, "Index of second antenna")); addFrequencyColumn(tableDesc); tableDesc.addColumn(casacore::ScalarColumnDesc( ColumnNameKind, "Index of the statistic kind")); addValueColumn(tableDesc, polarizationCount); casacore::SetupNewTable newTableSetup( TableToFilename(BaselineTimeStatisticTable), tableDesc, casacore::Table::New); const casacore::Table newTable(newTableSetup); openMainTable(true); _measurementSet->rwKeywordSet().defineTable( TableToName(BaselineTimeStatisticTable), newTable); } unsigned QualityTablesFormatter::StoreKindName(const std::string& name) { // This should be done atomically, but two quality writers in the same table // would be a weird thing to do anyway, plus I don't know how the casa tables // can be made atomic (and still have good performance). openKindNameTable(true); const unsigned kindIndex = findFreeKindIndex(*_kindNameTable); const unsigned newRow = _kindNameTable->nrow(); _kindNameTable->addRow(); casacore::ScalarColumn kindColumn(*_kindNameTable, ColumnNameKind); casacore::ScalarColumn nameColumn(*_kindNameTable, ColumnNameName); kindColumn.put(newRow, kindIndex); nameColumn.put(newRow, name); return kindIndex; } unsigned QualityTablesFormatter::findFreeKindIndex(casacore::Table& kindTable) { int maxIndex = 0; const casacore::ROScalarColumn kindColumn(kindTable, ColumnNameKind); const unsigned nrRow = kindTable.nrow(); for (unsigned i = 0; i < nrRow; ++i) { if (kindColumn(i) > maxIndex) maxIndex = kindColumn(i); } return maxIndex + 1; } void QualityTablesFormatter::openTable( QualityTable table, bool needWrite, std::unique_ptr& tablePtr) { if (tablePtr == nullptr) { openMainTable(false); tablePtr.reset(new casacore::Table( _measurementSet->keywordSet().asTable(TableToName(table)))); if (needWrite) tablePtr->reopenRW(); } else { if (needWrite && !tablePtr->isWritable()) tablePtr->reopenRW(); } } void QualityTablesFormatter::StoreTimeValue(double time, double frequency, const StatisticalValue& value) { openTimeTable(true); const unsigned newRow = _timeTable->nrow(); _timeTable->addRow(); // TODO maybe the columns need to be cached to avoid having to look them up // for each store... casacore::ScalarColumn timeColumn(*_timeTable, ColumnNameTime); casacore::ScalarColumn frequencyColumn(*_timeTable, ColumnNameFrequency); casacore::ScalarColumn kindColumn(*_timeTable, ColumnNameKind); casacore::ArrayColumn valueColumn(*_timeTable, ColumnNameValue); timeColumn.put(newRow, time); frequencyColumn.put(newRow, frequency); kindColumn.put(newRow, value.KindIndex()); casacore::Vector data(value.PolarizationCount()); for (unsigned i = 0; i < value.PolarizationCount(); ++i) data[i] = value.Value(i); valueColumn.put(newRow, data); } void QualityTablesFormatter::StoreFrequencyValue( double frequency, const StatisticalValue& value) { openFrequencyTable(true); const unsigned newRow = _frequencyTable->nrow(); _frequencyTable->addRow(); casacore::ScalarColumn frequencyColumn(*_frequencyTable, ColumnNameFrequency); casacore::ScalarColumn kindColumn(*_frequencyTable, ColumnNameKind); casacore::ArrayColumn valueColumn(*_frequencyTable, ColumnNameValue); frequencyColumn.put(newRow, frequency); kindColumn.put(newRow, value.KindIndex()); casacore::Vector data(value.PolarizationCount()); for (unsigned i = 0; i < value.PolarizationCount(); ++i) data[i] = value.Value(i); valueColumn.put(newRow, data); } void QualityTablesFormatter::StoreBaselineValue(unsigned antenna1, unsigned antenna2, double frequency, const StatisticalValue& value) { openBaselineTable(true); const unsigned newRow = _baselineTable->nrow(); _baselineTable->addRow(); casacore::ScalarColumn antenna1Column(*_baselineTable, ColumnNameAntenna1); casacore::ScalarColumn antenna2Column(*_baselineTable, ColumnNameAntenna2); casacore::ScalarColumn frequencyColumn(*_baselineTable, ColumnNameFrequency); casacore::ScalarColumn kindColumn(*_baselineTable, ColumnNameKind); casacore::ArrayColumn valueColumn(*_baselineTable, ColumnNameValue); antenna1Column.put(newRow, antenna1); antenna2Column.put(newRow, antenna2); frequencyColumn.put(newRow, frequency); kindColumn.put(newRow, value.KindIndex()); casacore::Vector data(value.PolarizationCount()); for (unsigned i = 0; i < value.PolarizationCount(); ++i) data[i] = value.Value(i); valueColumn.put(newRow, data); } void QualityTablesFormatter::StoreBaselineTimeValue( unsigned antenna1, unsigned antenna2, double time, double frequency, const StatisticalValue& value) { openBaselineTimeTable(true); const unsigned newRow = _baselineTimeTable->nrow(); _baselineTimeTable->addRow(); casacore::ScalarColumn timeColumn(*_baselineTimeTable, ColumnNameTime); casacore::ScalarColumn antenna1Column(*_baselineTimeTable, ColumnNameAntenna1); casacore::ScalarColumn antenna2Column(*_baselineTimeTable, ColumnNameAntenna2); casacore::ScalarColumn frequencyColumn(*_baselineTimeTable, ColumnNameFrequency); casacore::ScalarColumn kindColumn(*_baselineTimeTable, ColumnNameKind); casacore::ArrayColumn valueColumn(*_baselineTimeTable, ColumnNameValue); timeColumn.put(newRow, time); antenna1Column.put(newRow, antenna1); antenna2Column.put(newRow, antenna2); frequencyColumn.put(newRow, frequency); kindColumn.put(newRow, value.KindIndex()); casacore::Vector data(value.PolarizationCount()); for (unsigned i = 0; i < value.PolarizationCount(); ++i) data[i] = value.Value(i); valueColumn.put(newRow, data); } void QualityTablesFormatter::removeStatisticFromStatTable( enum QualityTable qualityTable, enum StatisticKind kind) { unsigned kindIndex; if (QueryKindIndex(kind, kindIndex)) { casacore::Table& table = getTable(qualityTable, true); const casacore::ScalarColumn kindColumn(table, ColumnNameKind); unsigned nrRow = table.nrow(); for (unsigned i = 0; i < nrRow; ++i) { while (i < nrRow && kindColumn(i) == (int)kindIndex) { table.removeRow(i); --nrRow; } } } } void QualityTablesFormatter::removeKindNameEntry(enum StatisticKind kind) { openKindNameTable(true); const casacore::ScalarColumn nameColumn(*_kindNameTable, ColumnNameName); const unsigned nrRow = _kindNameTable->nrow(); const casacore::String kindName(KindToName(kind)); for (unsigned i = 0; i < nrRow; ++i) { if (nameColumn(i) == kindName) { _kindNameTable->removeRow(i); break; } } } void QualityTablesFormatter::removeEntries(enum QualityTable table) { casacore::Table& casaTable = getTable(table, true); const unsigned nrRow = casaTable.nrow(); for (int i = nrRow - 1; i >= 0; --i) { casaTable.removeRow(i); } } unsigned QualityTablesFormatter::QueryStatisticEntryCount( enum StatisticDimension dimension, unsigned kindIndex) { const casacore::Table& casaTable( getTable(DimensionToTable(dimension), false)); const casacore::ROScalarColumn kindColumn(casaTable, ColumnNameKind); const unsigned nrRow = casaTable.nrow(); size_t count = 0; for (unsigned i = 0; i < nrRow; ++i) { if (kindColumn(i) == (int)kindIndex) ++count; } return count; } unsigned QualityTablesFormatter::GetPolarizationCount() { const casacore::Table& table(getTable(TimeStatisticTable, false)); const casacore::ROArrayColumn valueColumn(table, ColumnNameValue); return valueColumn.columnDesc().shape()[0]; } void QualityTablesFormatter::QueryTimeStatistic( unsigned kindIndex, std::vector>& entries) { const casacore::Table& table(getTable(TimeStatisticTable, false)); const unsigned nrRow = table.nrow(); const casacore::ROScalarColumn timeColumn(table, ColumnNameTime); const casacore::ROScalarColumn frequencyColumn(table, ColumnNameFrequency); const casacore::ROScalarColumn kindColumn(table, ColumnNameKind); const casacore::ROArrayColumn valueColumn(table, ColumnNameValue); const int polarizationCount = valueColumn.columnDesc().shape()[0]; for (unsigned i = 0; i < nrRow; ++i) { if (kindColumn(i) == (int)kindIndex) { StatisticalValue value(polarizationCount); value.SetKindIndex(kindIndex); casacore::Array valueArray = valueColumn(i); casacore::Array::iterator iter = valueArray.begin(); for (int p = 0; p < polarizationCount; ++p) { value.SetValue(p, *iter); ++iter; } TimePosition position; position.time = timeColumn(i); position.frequency = frequencyColumn(i); entries.push_back( std::pair(position, value)); } } } void QualityTablesFormatter::QueryFrequencyStatistic( unsigned kindIndex, std::vector>& entries) { const casacore::Table& table(getTable(FrequencyStatisticTable, false)); const unsigned nrRow = table.nrow(); const casacore::ROScalarColumn frequencyColumn(table, ColumnNameFrequency); const casacore::ROScalarColumn kindColumn(table, ColumnNameKind); const casacore::ROArrayColumn valueColumn(table, ColumnNameValue); const int polarizationCount = valueColumn.columnDesc().shape()[0]; for (unsigned i = 0; i < nrRow; ++i) { if (kindColumn(i) == (int)kindIndex) { StatisticalValue value(polarizationCount); value.SetKindIndex(kindIndex); casacore::Array valueArray = valueColumn(i); casacore::Array::iterator iter = valueArray.begin(); for (int p = 0; p < polarizationCount; ++p) { value.SetValue(p, *iter); ++iter; } FrequencyPosition position; position.frequency = frequencyColumn(i); entries.push_back( std::pair(position, value)); } } } void QualityTablesFormatter::QueryBaselineStatistic( unsigned kindIndex, std::vector>& entries) { const casacore::Table& table(getTable(BaselineStatisticTable, false)); const unsigned nrRow = table.nrow(); const casacore::ROScalarColumn antenna1Column(table, ColumnNameAntenna1); const casacore::ROScalarColumn antenna2Column(table, ColumnNameAntenna2); const casacore::ROScalarColumn frequencyColumn(table, ColumnNameFrequency); const casacore::ROScalarColumn kindColumn(table, ColumnNameKind); const casacore::ROArrayColumn valueColumn(table, ColumnNameValue); const int polarizationCount = valueColumn.columnDesc().shape()[0]; for (unsigned i = 0; i < nrRow; ++i) { if (kindColumn(i) == (int)kindIndex) { StatisticalValue value(polarizationCount); value.SetKindIndex(kindIndex); casacore::Array valueArray = valueColumn(i); casacore::Array::iterator iter = valueArray.begin(); for (int p = 0; p < polarizationCount; ++p) { value.SetValue(p, *iter); ++iter; } BaselinePosition position; position.antenna1 = antenna1Column(i); position.antenna2 = antenna2Column(i); position.frequency = frequencyColumn(i); entries.push_back( std::pair(position, value)); } } } void QualityTablesFormatter::openMainTable(bool needWrite) { if (_measurementSet == nullptr) { if (needWrite) _measurementSet.reset( new casacore::Table(_measurementSetName, casacore::Table::Update)); else _measurementSet.reset(new casacore::Table(_measurementSetName)); } else if (needWrite) { if (!_measurementSet->isWritable()) _measurementSet->reopenRW(); } } aoflagger-v3.5.1/quality/loghistogram.h0000664000175000017500000003546614752462134016345 0ustar oleole#ifndef LOGHISTOGRAM_H #define LOGHISTOGRAM_H #include #include #include #include #include "histogramtablesformatter.h" #include "../util/serializable.h" #ifndef HAVE_EXP10 #define exp10(x) exp((2.3025850929940456840179914546844) * (x)) #endif class LogHistogram : public Serializable { private: class AmplitudeBin : public Serializable { public: AmplitudeBin() : count(0) {} long unsigned count; long unsigned GetCount() const { return count; } AmplitudeBin& operator+=(const AmplitudeBin& other) { count += other.count; return *this; } AmplitudeBin& operator-=(const AmplitudeBin& other) { if (count >= other.count) count -= other.count; else count = 0; return *this; } virtual void Serialize(std::ostream& stream) const final override { SerializeToUInt64(stream, count); } virtual void Unserialize(std::istream& stream) final override { count = UnserializeUInt64(stream); } }; public: LogHistogram() {} LogHistogram(const LogHistogram& source) : _amplitudes(source._amplitudes) {} void Add(const double amplitude) { if (std::isfinite(amplitude)) { const double centralAmp = getCentralAmplitude(amplitude); AmplitudeBin& bin = getBin(centralAmp); ++bin.count; } } void Add(const LogHistogram& histogram) { for (std::map::const_iterator i = histogram._amplitudes.begin(); i != histogram._amplitudes.end(); ++i) { AmplitudeBin& bin = getBin(i->first); bin += i->second; } } void operator-=(const LogHistogram& histogram) { for (std::map::const_iterator i = histogram._amplitudes.begin(); i != histogram._amplitudes.end(); ++i) { AmplitudeBin& bin = getBin(i->first); bin -= i->second; } } double MaxAmplitude() const { if (_amplitudes.empty()) return 0.0; return _amplitudes.rbegin()->first; } double MinPositiveAmplitude() const { std::map::const_iterator i = _amplitudes.begin(); if (i == _amplitudes.end()) return 0.0; while (i->first <= 0.0) { ++i; if (i == _amplitudes.end()) return 0.0; } return i->first; } double NormalizedCount(double startAmplitude, double endAmplitude) const { unsigned long count = 0; for (std::map::const_iterator i = _amplitudes.begin(); i != _amplitudes.end(); ++i) { if (i->first >= startAmplitude && i->first < endAmplitude) count += i->second.GetCount(); } return (double)count / (endAmplitude - startAmplitude); } double NormalizedCount(double centreAmplitude) const { const double key = getCentralAmplitude(centreAmplitude); std::map::const_iterator i = _amplitudes.find(key); if (i == _amplitudes.end()) return 0.0; return (double)i->second.GetCount() / (binEnd(centreAmplitude) - binStart(centreAmplitude)); } double MinNormalizedCount() const { const_iterator i = begin(); if (i == end()) return 0.0; double minCount = i.normalizedCount(); do { const double c = i.normalizedCount(); if (c < minCount) minCount = c; ++i; } while (i != end()); return minCount; } double MaxNormalizedCount() const { double maxCount = 0.0; for (LogHistogram::const_iterator i = begin(); i != end(); ++i) { if (i.normalizedCount() > maxCount && i.value() > 0 && std::isfinite(i.value())) maxCount = i.normalizedCount(); } return maxCount; } double NormalizedTotalCount() const { unsigned long count = 0; for (LogHistogram::const_iterator i = begin(); i != end(); ++i) count += i.unnormalizedCount(); return count; } double NormalizedCountAbove(double lowerLimitingAmplitude) const { unsigned long count = 0; LogHistogram::const_iterator i = begin(); while (i != end() && i.value() <= lowerLimitingAmplitude) { ++i; } while (i != end()) { count += i.unnormalizedCount(); ++i; } return count; } double AmplitudeWithMaxNormalizedCount() const { double maxCount = 0.0, maxPosition = 0.0; for (LogHistogram::const_iterator i = begin(); i != end(); ++i) { if (i.normalizedCount() > maxCount && i.value() > 0 && std::isfinite(i.value())) { maxCount = i.normalizedCount(); maxPosition = i.value(); } } return maxPosition; } double MinPosNormalizedCount() const { const_iterator i = begin(); if (i == end()) return 0.0; double minCount = std::isfinite(i.normalizedCount()) ? i.normalizedCount() + 1.0 : 1.0; do { const double c = i.normalizedCount(); if (c < minCount && c > 0.0 && std::isfinite(c)) minCount = c; ++i; } while (i != end()); return minCount; } double NormalizedSlope(double startAmplitude, double endAmplitude) const { unsigned long n = 0; long double sumX = 0.0, sumXY = 0.0, sumY = 0.0, sumXSquare = 0.0; for (const_iterator i = begin(); i != end(); ++i) { if (i.value() >= startAmplitude && i.value() < endAmplitude) { double x = log10(i.value()); double y = log10(i.normalizedCount()); ++n; sumX += x; sumXSquare += x * x; sumY += y; sumXY += x * y; } } return (sumXY - sumX * sumY / n) / (sumXSquare - (sumX * sumX / n)); } double PowerLawExponent(double startAmplitude) const { const long double xMin = startAmplitude; long double termSum = 0.0; long double termCount = 0.0; for (const_iterator i = begin(); i != end(); ++i) { const long double x = i.value(); if (x >= startAmplitude) { const long double count = i.unnormalizedCount(); const long double thisTerm = logl(x / xMin); termCount += count; termSum += thisTerm * count; } } return (double)(-1.0L - termCount / termSum); } double PowerLawExponentStdError(double startAmplitude, double expEstimator) const { long double termCount = 0.0; for (const_iterator i = begin(); i != end(); ++i) { if (i.value() >= startAmplitude) { const long double count = i.unnormalizedCount(); termCount += count; } } return (double)((-expEstimator - 1.0L) / sqrtl(termCount)); } double NormalizedSlopeOffset(double startAmplitude, double endAmplitude, double slope) const { unsigned long n = 0; long double sumOffset = 0.0; for (const_iterator i = begin(); i != end(); ++i) { if (i.value() >= startAmplitude && i.value() < endAmplitude) { double y = log10(i.normalizedCount()); double x = log10(i.value()); double ySlope = x * slope; ++n; sumOffset += (y - ySlope); } } return (double)(sumOffset / (long double)n); } double NormalizedSlopeStdError(double startAmplitude, double endAmplitude, double slope) const { long double ssxx = 0.0, ssxy = 0.0, ssyy = 0.0, xSum = 0.0, ySum = 0.0; unsigned long n = 0; // determine the 'average' x and y for (const_iterator i = begin(); i != end(); ++i) { if (i.value() >= startAmplitude && i.value() < endAmplitude) { xSum += log10(i.value()); ySum += log10(i.normalizedCount()); ++n; } } const long double avgX = xSum / (long double)n, avgY = ySum / (long double)n; for (const_iterator i = begin(); i != end(); ++i) { if (i.value() >= startAmplitude && i.value() < endAmplitude) { long double y = log10(i.normalizedCount()); long double x = log10(i.value()); ssxx += (x - avgX) * (x - avgX); ssxy += (x - avgX) * (y - avgY); ssyy += (y - avgY) * (y - avgY); } } return (double)sqrtl((ssyy - slope * ssxy) / (ssxx * (long double)(n - 2))); } double NormalizedSlopeStdDevBySampling(double startAmplitude, double endAmplitude, double slope, double stepFactor) const { long double sum = 0.0; unsigned long n = 0; if (stepFactor <= 1.0001) stepFactor = 1.0001; while (startAmplitude < endAmplitude) { const double stepEnd = startAmplitude * stepFactor; double sampledSlope = NormalizedSlope(startAmplitude, stepEnd); double sampleError = sampledSlope - slope; sum += sampleError * sampleError; ++n; startAmplitude = stepEnd; } return (double)sqrtl(sum / ((long double)n * n - n)); } double PowerLawUpperLimit(double constrainingAmplitude, double exponent, double factor) const { const double count = NormalizedCountAbove(constrainingAmplitude); const double term = count * (exponent + 1.0) / factor + pow(constrainingAmplitude, exponent + 1.0); return pow(term, 1.0 / (exponent + 1.0)); } double PowerLawLowerLimit(double constrainingAmplitude, double exponent, double factor, double rfiRatio) const { const double countPart = NormalizedCountAbove(constrainingAmplitude); const double countTotal = NormalizedTotalCount() * rfiRatio; const double term = (countPart - countTotal) * (exponent + 1.0) / factor + pow(constrainingAmplitude, exponent + 1.0); return pow(term, 1.0 / (exponent + 1.0)); } double PowerLawLowerLimit2(double constrainingAmplitude, double exponent, double factor, double rfiRatio) const { const double countPart = NormalizedCountAbove(constrainingAmplitude); const double countTotal = NormalizedTotalCount() * rfiRatio; const double term = (countPart - countTotal) * (exponent + 1.0) / factor + pow(constrainingAmplitude, exponent + 1.0); return pow(term / -exponent, 1.0 / (exponent + 1.0)); } void GetRFIRegion(double& start, double& end) const { double sigmaEstimate = AmplitudeWithMaxNormalizedCount(); double maxAmplitude = MaxAmplitude(); start = sigmaEstimate * 20.0; double halfWay = exp((log(start) + log(maxAmplitude)) * 0.5); end = halfWay; } double NormalizedSlopeInRFIRegion() const { double start, end; GetRFIRegion(start, end); return NormalizedSlope(start, end); } void SetData( std::vector& histogramData) { for (std::vector::const_iterator i = histogramData.begin(); i != histogramData.end(); ++i) { const double b = (i->binStart + i->binEnd) * 0.5; // TODO somewhat inefficient... getBin(getCentralAmplitude(b)).count = (unsigned long)i->count; } } void Rescale(double factor) { std::map newAmplitudes; std::map::iterator position = newAmplitudes.begin(); for (std::map::const_iterator i = _amplitudes.begin(); i != _amplitudes.end(); ++i) { position = newAmplitudes.insert( position, std::pair( getCentralAmplitude(i->first * factor), i->second)); } _amplitudes = newAmplitudes; } class const_iterator { public: const_iterator(const LogHistogram& histogram, std::map::const_iterator iter) : _iterator(iter) {} const_iterator(const const_iterator& source) : _iterator(source._iterator) {} const_iterator& operator=(const const_iterator& source) { _iterator = source._iterator; return *this; } bool operator==(const const_iterator& other) const { return other._iterator == _iterator; } bool operator!=(const const_iterator& other) const { return other._iterator != _iterator; } const_iterator& operator++() { ++_iterator; return *this; } const_iterator& operator--() { --_iterator; return *this; } double value() const { return _iterator->first; } double normalizedCount() const { return _iterator->second.GetCount() / (binEnd() - binStart()); } long unsigned unnormalizedCount() const { return _iterator->second.GetCount(); } double binStart() const { return _iterator->first > 0.0 ? exp10(log10(_iterator->first) - 0.005) : -exp10(log10(-_iterator->first) - 0.005); } double binEnd() const { return _iterator->first > 0.0 ? exp10(log10(_iterator->first) + 0.005) : -exp10(log10(-_iterator->first) + 0.005); } private: std::map::const_iterator _iterator; }; typedef const_iterator iterator; const_iterator begin() const { return const_iterator(*this, _amplitudes.begin()); } const_iterator end() const { return const_iterator(*this, _amplitudes.end()); } virtual void Serialize(std::ostream& stream) const final override { SerializeToUInt64(stream, _amplitudes.size()); for (std::map::const_iterator i = _amplitudes.begin(); i != _amplitudes.end(); ++i) { SerializeToDouble(stream, i->first); i->second.Serialize(stream); } } virtual void Unserialize(std::istream& stream) final override { _amplitudes.clear(); size_t mapSize = UnserializeUInt64(stream); std::map::iterator insertPos = _amplitudes.begin(); for (size_t i = 0; i != mapSize; ++i) { std::pair p; p.first = UnserializeDouble(stream); p.second.Unserialize(stream); insertPos = _amplitudes.insert(insertPos, p); } } void CreateMissingBins() { double first = MinPositiveAmplitude(), last = MaxAmplitude(); for (double i = first; i < last; i *= 1.01) { getBin(getCentralAmplitude(i)); } } private: std::map _amplitudes; AmplitudeBin& getBin(double centralAmplitude) { std::map::iterator element = _amplitudes.find(centralAmplitude); if (element == _amplitudes.end()) { element = _amplitudes .insert(std::pair(centralAmplitude, AmplitudeBin())) .first; } return element->second; } double binStart(double x) const { return x > 0.0 ? exp10(log10(x) - 0.005) : -exp10(log10(x) - 0.005); } double binEnd(double x) const { return x > 0.0 ? exp10(log10(x) + 0.005) : -exp10(log10(x) + 0.005); } static double getCentralAmplitude(const double amplitude) { if (amplitude >= 0.0) return exp10(round(100.0 * log10(amplitude)) / 100.0); else return -exp10(round(100.0 * log10(-amplitude)) / 100.0); } }; #endif aoflagger-v3.5.1/quality/histogramcollection.h0000664000175000017500000002516014752462134017705 0ustar oleole#ifndef HISTOGRAM_COLLECTION_H #define HISTOGRAM_COLLECTION_H #include "loghistogram.h" #include #include #include #include "../structures/image2d.h" #include "../structures/mask2d.h" #include "../util/serializable.h" class HistogramCollection : public Serializable { public: typedef std::pair AntennaPair; HistogramCollection() : _polarizationCount(0), _totalHistograms(nullptr), _rfiHistograms(nullptr) {} explicit HistogramCollection(unsigned polarizationCount) : _polarizationCount(polarizationCount) { init(); } HistogramCollection(const HistogramCollection& source) : _polarizationCount(source._polarizationCount) { init(); for (unsigned i = 0; i < _polarizationCount; ++i) { copy(_totalHistograms[i], source._totalHistograms[i]); copy(_rfiHistograms[i], source._rfiHistograms[i]); } } ~HistogramCollection() { destruct(); } void SetPolarizationCount(unsigned polarizationCount) { destruct(); _polarizationCount = polarizationCount; init(); } void Add(const unsigned antenna1, const unsigned antenna2, const unsigned polarization, const std::complex* values, const bool* isRFI, size_t sampleCount) { LogHistogram& totalHistogram = GetTotalHistogram(antenna1, antenna2, polarization); LogHistogram& rfiHistogram = GetRFIHistogram(antenna1, antenna2, polarization); for (size_t i = 0; i < sampleCount; ++i) { const double amplitude = sqrt(values[i].real() * values[i].real() + values[i].imag() * values[i].imag()); totalHistogram.Add(amplitude); if (isRFI[i]) rfiHistogram.Add(amplitude); } } void Add(const HistogramCollection& collection) { if (collection._polarizationCount != _polarizationCount) throw std::runtime_error( "Polarization counts of histogram collections don't match"); for (unsigned p = 0; p < _polarizationCount; ++p) { add(collection, p, p); } } bool Empty() const { if (_polarizationCount == 0) return true; for (unsigned p = 0; p != _polarizationCount; ++p) { if (!_totalHistograms[p].empty() || !_rfiHistograms[p].empty()) return false; } return true; } void Add(const unsigned antenna1, const unsigned antenna2, const unsigned polarization, Image2DCPtr image, Mask2DCPtr mask); void Add(const unsigned antenna1, const unsigned antenna2, const unsigned polarization, Image2DCPtr image, Mask2DCPtr flagMask, Mask2DCPtr correlatorMask); void Add(const unsigned antenna1, const unsigned antenna2, const unsigned polarization, Image2DCPtr real, Image2DCPtr imaginary, Mask2DCPtr flagMask, Mask2DCPtr correlatorMask); LogHistogram& GetTotalHistogram(const unsigned a1, const unsigned a2, const unsigned polarization) { return getHistogram(_totalHistograms, a1, a2, polarization); } LogHistogram& GetRFIHistogram(const unsigned a1, const unsigned a2, const unsigned polarization) { return getHistogram(_rfiHistograms, a1, a2, polarization); } const std::map& GetTotalHistogram( const unsigned polarization) const { return _totalHistograms[polarization]; } const std::map& GetRFIHistogram( const unsigned polarization) const { return _rfiHistograms[polarization]; } void GetTotalHistogramForCrossCorrelations(const unsigned polarization, LogHistogram& target) const { getHistogramForCrossCorrelations(_totalHistograms, polarization, target); } void GetRFIHistogramForCrossCorrelations(const unsigned polarization, LogHistogram& target) const { getHistogramForCrossCorrelations(_rfiHistograms, polarization, target); } void Clear() { destruct(); init(); } void Save(class HistogramTablesFormatter& histogramTables); void Load(class HistogramTablesFormatter& histogramTables); unsigned PolarizationCount() const { return _polarizationCount; } virtual void Serialize(std::ostream& stream) const final override { SerializeToUInt64(stream, _polarizationCount); serializeMapArray(stream, _totalHistograms); serializeMapArray(stream, _rfiHistograms); } virtual void Unserialize(std::istream& stream) final override { destruct(); _polarizationCount = UnserializeUInt64(stream); init(); unserializeMapArray(stream, _totalHistograms); unserializeMapArray(stream, _rfiHistograms); } HistogramCollection* CreateSummedPolarizationCollection() const { HistogramCollection* newCollection = new HistogramCollection(1); for (unsigned p = 0; p < _polarizationCount; ++p) newCollection->add(*this, p, 0); return newCollection; } void Rescale(double factor) { for (unsigned p = 0; p < _polarizationCount; ++p) { for (std::map::iterator i = _totalHistograms[p].begin(); i != _totalHistograms[p].end(); ++i) { i->second->Rescale(factor); } for (std::map::iterator i = _rfiHistograms[p].begin(); i != _rfiHistograms[p].end(); ++i) { i->second->Rescale(factor); } } } void CreateMissingBins() { for (unsigned p = 0; p < _polarizationCount; ++p) { for (std::map::iterator i = _totalHistograms[p].begin(); i != _totalHistograms[p].end(); ++i) { i->second->CreateMissingBins(); } for (std::map::iterator i = _rfiHistograms[p].begin(); i != _rfiHistograms[p].end(); ++i) { i->second->CreateMissingBins(); } } } private: unsigned _polarizationCount; std::map* _totalHistograms; std::map* _rfiHistograms; void init() { if (_polarizationCount != 0) { _totalHistograms = new std::map[_polarizationCount]; _rfiHistograms = new std::map[_polarizationCount]; } else { _totalHistograms = nullptr; _rfiHistograms = nullptr; } } void destruct() { if (_polarizationCount != 0) { for (unsigned p = 0; p < _polarizationCount; ++p) { for (std::map::iterator i = _totalHistograms[p].begin(); i != _totalHistograms[p].end(); ++i) { delete i->second; } for (std::map::iterator i = _rfiHistograms[p].begin(); i != _rfiHistograms[p].end(); ++i) { delete i->second; } } delete[] _totalHistograms; delete[] _rfiHistograms; } } void serializeMapArray( std::ostream& stream, const std::map* map) const { for (unsigned p = 0; p < _polarizationCount; ++p) serializeMap(stream, map[p]); } void unserializeMapArray(std::istream& stream, std::map* map) { for (unsigned p = 0; p < _polarizationCount; ++p) unserializeMap(stream, map[p]); } void serializeMap(std::ostream& stream, const std::map& map) const { SerializeToUInt64(stream, map.size()); for (std::map::const_iterator i = map.begin(); i != map.end(); ++i) { const AntennaPair& antennae = i->first; const LogHistogram* histogram = i->second; SerializeToUInt32(stream, antennae.first); SerializeToUInt32(stream, antennae.second); histogram->Serialize(stream); } } void unserializeMap(std::istream& stream, std::map& map) { map.clear(); size_t mapSize = UnserializeUInt64(stream); std::map::iterator insertPos = map.begin(); for (size_t i = 0; i != mapSize; ++i) { std::pair p; p.first.first = UnserializeUInt32(stream); p.first.second = UnserializeUInt32(stream); p.second = new LogHistogram(); p.second->Unserialize(stream); insertPos = map.insert(insertPos, p); } } void copy(std::map& destination, const std::map& source) { for (std::map::const_iterator i = source.begin(); i != source.end(); ++i) { destination.insert(std::pair( i->first, new LogHistogram(*i->second))); } } void add(const HistogramCollection& collection, unsigned fromPolarization, unsigned toPolarization) { for (std::map::const_iterator i = collection._totalHistograms[fromPolarization].begin(); i != collection._totalHistograms[fromPolarization].end(); ++i) { LogHistogram& histogram = GetTotalHistogram(i->first.first, i->first.second, toPolarization); histogram.Add(*i->second); } for (std::map::const_iterator i = collection._rfiHistograms[fromPolarization].begin(); i != collection._rfiHistograms[fromPolarization].end(); ++i) { LogHistogram& histogram = GetRFIHistogram(i->first.first, i->first.second, toPolarization); histogram.Add(*i->second); } } LogHistogram& getHistogram(std::map* histograms, const unsigned a1, const unsigned a2, const unsigned polarization) { const AntennaPair antennae(a1, a2); std::map::iterator i = histograms[polarization].find(antennae); if (i == histograms[polarization].end()) { i = histograms[polarization] .insert(std::pair(antennae, new LogHistogram())) .first; } return *i->second; } void getHistogramForCrossCorrelations( std::map* histograms, const unsigned polarization, LogHistogram& target) const { for (std::map::const_iterator i = histograms[polarization].begin(); i != histograms[polarization].end(); ++i) { if (i->first.first != i->first.second) target.Add(*i->second); } } }; #endif aoflagger-v3.5.1/quality/statisticscollection.h0000664000175000017500000010401214752462134020074 0ustar oleole#ifndef STATISTICS_COLLECTION_H #define STATISTICS_COLLECTION_H #include #include "../structures/image2d.h" #include "../structures/mask2d.h" #include "../util/serializable.h" #include "baselinestatisticsmap.h" #include "defaultstatistics.h" #include "qualitytablesformatter.h" #include "statisticalvalue.h" #include class StatisticsCollection : public Serializable { private: typedef std::map DoubleStatMap; public: StatisticsCollection() : _polarizationCount(0), _emptyBaselineStatisticsMap(0) {} explicit StatisticsCollection(unsigned polarizationCount) : _polarizationCount(polarizationCount), _emptyBaselineStatisticsMap(polarizationCount) {} StatisticsCollection(const StatisticsCollection& source) : _timeStatistics(source._timeStatistics), _frequencyStatistics(source._frequencyStatistics), _baselineStatistics(source._baselineStatistics), _polarizationCount(source._polarizationCount), _emptyBaselineStatisticsMap(source._polarizationCount) {} StatisticsCollection& operator=(const StatisticsCollection& source) { _timeStatistics = source._timeStatistics; _frequencyStatistics = source._frequencyStatistics; _baselineStatistics = source._baselineStatistics; _polarizationCount = source._polarizationCount; _emptyBaselineStatisticsMap = source._emptyBaselineStatisticsMap; return *this; } void Clear() { _timeStatistics.clear(); _frequencyStatistics.clear(); _baselineStatistics.clear(); } bool HasBand(unsigned band) const { return _bands.find(band) != _bands.end(); } void InitializeBand(unsigned band, const double* frequencies, unsigned channelCount) { std::vector pointers; for (unsigned i = 0; i < channelCount; ++i) { pointers.emplace_back(&getFrequencyStatistic(frequencies[i])); } _bands.emplace(band, pointers); double centralFrequency = (frequencies[0] + frequencies[channelCount - 1]) / 2.0; _centralFrequencies.emplace(band, centralFrequency); _bandFrequencies.emplace( band, std::vector(frequencies, frequencies + channelCount)); } void Add(unsigned antenna1, unsigned antenna2, double time, unsigned band, int polarization, const float* reals, const float* imags, const bool* isRFI, const bool* origFlags, unsigned nsamples, unsigned step, unsigned stepRFI, unsigned stepFlags); void Add(unsigned antenna1, unsigned antenna2, double time, unsigned band, int polarization, const std::vector>& samples, const bool* isRFI) { const float* dataPtr = reinterpret_cast(&(samples[0])); bool origFlag = false; Add(antenna1, antenna2, time, band, polarization, dataPtr, dataPtr + 1, // real and imag parts isRFI, &origFlag, samples.size(), 2, 1, 0); } void AddToTimeFrequency(unsigned antenna1, unsigned antenna2, double time, unsigned band, int polarization, const float* reals, const float* imags, const bool* isRFI, const bool* origFlags, unsigned nsamples, unsigned step, unsigned stepRFI, unsigned stepFlags); void AddImage(unsigned antenna1, unsigned antenna2, const double* times, unsigned band, int polarization, const Image2DCPtr& realImage, const Image2DCPtr& imagImage, const Mask2DCPtr& rfiMask, const Mask2DCPtr& correlatorMask); void Save(QualityTablesFormatter& qualityData) const { saveTime(qualityData); saveFrequency(qualityData); saveBaseline(qualityData); } void Load(QualityTablesFormatter& qualityData) { SetPolarizationCount(qualityData.GetPolarizationCount()); loadTime(qualityData); loadFrequency(qualityData); loadBaseline(qualityData); } void LoadTimeStatisticsOnly(QualityTablesFormatter& qualityData) { loadTime(qualityData); } void Add(QualityTablesFormatter& qualityData) { loadTime(qualityData); loadFrequency(qualityData); loadBaseline(qualityData); } void Add(const StatisticsCollection& collection) { addTime(collection); addFrequency(collection); addBaseline(collection); } void GetGlobalTimeStatistics(DefaultStatistics& statistics) const { statistics = getGlobalStatistics(_timeStatistics); } void GetGlobalFrequencyStatistics(DefaultStatistics& statistics) const { statistics = getGlobalStatistics(_frequencyStatistics); } void GetGlobalAutoBaselineStatistics(DefaultStatistics& statistics) const { statistics = getGlobalBaselineStatistics(); } void GetGlobalCrossBaselineStatistics(DefaultStatistics& statistics) const { statistics = getGlobalBaselineStatistics(); } void GetFrequencyRangeStatistics(DefaultStatistics& statistics, double startFrequency, double endFrequency) const { statistics = getFrequencyRangeStatistics(startFrequency, endFrequency); } const BaselineStatisticsMap& BaselineStatistics() const { if (_baselineStatistics.size() == 1) return _baselineStatistics.begin()->second; else if (_baselineStatistics.size() == 0) return _emptyBaselineStatisticsMap; else throw std::runtime_error( "Requesting single band single baseline statistics in statistics " "collection with multiple bands"); } const std::map& TimeStatistics() const { if (_timeStatistics.size() == 1) return _timeStatistics.begin()->second; else throw std::runtime_error( "Requesting single band single timestep statistics in statistics " "collection with multiple bands"); } const std::map>& AllTimeStatistics() const { return _timeStatistics; } const std::map& FrequencyStatistics() const { return _frequencyStatistics; } std::map GetAntennaStatistics() const { const BaselineStatisticsMap& map = BaselineStatistics(); std::vector> baselines = map.BaselineList(); std::map antStatistics; for (const std::pair& p : baselines) { if (p.first != p.second) { const DefaultStatistics& stats = map.GetStatistics(p.first, p.second); addAntennaStatistic(p.first, stats, antStatistics); addAntennaStatistic(p.second, stats, antStatistics); } } return antStatistics; } unsigned PolarizationCount() const { return _polarizationCount; } void SetPolarizationCount(unsigned newCount) { if (newCount != _polarizationCount) { _polarizationCount = newCount; _emptyBaselineStatisticsMap = BaselineStatisticsMap(_polarizationCount); _timeStatistics.clear(); _frequencyStatistics.clear(); _baselineStatistics.clear(); } } virtual void Serialize(std::ostream& stream) const final override { SerializeToUInt64(stream, _polarizationCount); serializeTime(stream); serializeFrequency(stream); serializeBaselines(stream); } virtual void Unserialize(std::istream& stream) final override { _polarizationCount = UnserializeUInt64(stream); _emptyBaselineStatisticsMap = BaselineStatisticsMap(_polarizationCount); unserializeTime(stream); unserializeFrequency(stream); unserializeBaselines(stream); } void IntegrateBaselinesToOneChannel() { const size_t size = _baselineStatistics.size(); if (size > 1) { BaselineStatisticsMap fullMap(_polarizationCount); double frequencySum = 0.0; for (std::map::const_iterator i = _baselineStatistics.begin(); i != _baselineStatistics.end(); ++i) { frequencySum += i->first; fullMap += i->second; } _baselineStatistics.clear(); _baselineStatistics.insert(std::pair( frequencySum / size, fullMap)); } } void IntegrateTimeToOneChannel() { const size_t size = _timeStatistics.size(); if (size > 1) { DoubleStatMap fullMap; double frequencySum = 0.0; for (std::map::const_iterator i = _timeStatistics.begin(); i != _timeStatistics.end(); ++i) { frequencySum += i->first; addToDoubleStatMap(fullMap, i->second); } _timeStatistics.clear(); _timeStatistics.insert( std::pair(frequencySum / size, fullMap)); } } void LowerTimeResolution(size_t maxSteps) { for (std::map::iterator i = _timeStatistics.begin(); i != _timeStatistics.end(); ++i) { lowerResolution(i->second, maxSteps); } } void LowerFrequencyResolution(size_t maxSteps) { lowerResolution(_frequencyStatistics, maxSteps); } /** * The regrid method will force all channels(/sub-bands) inside the collection * to have the same uniform grid. It will do this by moving around time steps, * using the first grid as reference. This is useful for raw (not NDPPP-ed) * data, that might contain slightly different time steps in the different * sub-bands, but are otherwise similarly gridded. */ void RegridTime() { if (_timeStatistics.size() > 1) { std::map::iterator i = _timeStatistics.begin(); const DoubleStatMap& referenceMap = i->second; ++i; do { regrid(referenceMap, i->second); ++i; } while (i != _timeStatistics.end()); } } private: struct StatisticSaver { QualityTablesFormatter::StatisticDimension dimension; double time; double frequency; unsigned antenna1; unsigned antenna2; QualityTablesFormatter* qualityData; void Save(StatisticalValue& value, unsigned kindIndex) { value.SetKindIndex(kindIndex); switch (dimension) { case QualityTablesFormatter::TimeDimension: qualityData->StoreTimeValue(time, frequency, value); break; case QualityTablesFormatter::FrequencyDimension: qualityData->StoreFrequencyValue(frequency, value); break; case QualityTablesFormatter::BaselineDimension: qualityData->StoreBaselineValue(antenna1, antenna2, frequency, value); break; case QualityTablesFormatter::BaselineTimeDimension: qualityData->StoreBaselineTimeValue(antenna1, antenna2, time, frequency, value); break; } } }; struct Indices { unsigned kindRFICount; unsigned kindCount; unsigned kindSum; unsigned kindSumP2; unsigned kindDCount; unsigned kindDSum; unsigned kindDSumP2; void fill(QualityTablesFormatter& qd) { kindRFICount = qd.StoreOrQueryKindIndex(QualityTablesFormatter::RFICountStatistic), kindCount = qd.StoreOrQueryKindIndex(QualityTablesFormatter::CountStatistic), kindSum = qd.StoreOrQueryKindIndex(QualityTablesFormatter::SumStatistic), kindSumP2 = qd.StoreOrQueryKindIndex(QualityTablesFormatter::SumP2Statistic), kindDCount = qd.StoreOrQueryKindIndex(QualityTablesFormatter::DCountStatistic), kindDSum = qd.StoreOrQueryKindIndex(QualityTablesFormatter::DSumStatistic), kindDSumP2 = qd.StoreOrQueryKindIndex(QualityTablesFormatter::DSumP2Statistic); } }; template void addTimeAndBaseline(unsigned antenna1, unsigned antenna2, double time, double centralFrequency, int polarization, const float* reals, const float* imags, const bool* isRFI, const bool* origFlags, unsigned nsamples, unsigned step, unsigned stepRFI, unsigned stepFlags); template void addToTimeFrequency(double time, const double* frequencies, int polarization, const float* reals, const float* imags, const bool* isRFI, const bool* origFlags, unsigned nsamples, unsigned step, unsigned stepRFI, unsigned stepFlags, bool shiftOneUp); template void addToStatistic(DefaultStatistics& statistic, unsigned polarization, unsigned long count, long double sum_R, long double sum_I, long double sumP2_R, long double sumP2_I, unsigned long rfiCount) { if (IsDiff) { statistic.dCount[polarization] += count; statistic.dSum[polarization] += std::complex(sum_R, sum_I); statistic.dSumP2[polarization] += std::complex(sumP2_R, sumP2_I); } else { statistic.count[polarization] += count; statistic.sum[polarization] += std::complex(sum_R, sum_I); statistic.sumP2[polarization] += std::complex(sumP2_R, sumP2_I); statistic.rfiCount[polarization] += rfiCount; } } template void addSingleNonRFISampleToStatistic(DefaultStatistics& statistic, unsigned polarization, long double sum_R, long double sum_I, long double sumP2_R, long double sumP2_I) { if (IsDiff) { ++statistic.dCount[polarization]; statistic.dSum[polarization] += std::complex(sum_R, sum_I); statistic.dSumP2[polarization] += std::complex(sumP2_R, sumP2_I); } else { ++statistic.count[polarization]; statistic.sum[polarization] += std::complex(sum_R, sum_I); statistic.sumP2[polarization] += std::complex(sumP2_R, sumP2_I); } } template void addFrequency(unsigned band, int polarization, const float* reals, const float* imags, const bool* isRFI, const bool* origFlags, unsigned nsamples, unsigned step, unsigned stepRFI, unsigned stepFlags, bool shiftOneUp); void initializeEmptyStatistics( QualityTablesFormatter& qualityData, QualityTablesFormatter::StatisticDimension dimension) const { qualityData.InitializeEmptyStatistic( dimension, QualityTablesFormatter::RFICountStatistic, _polarizationCount); qualityData.InitializeEmptyStatistic( dimension, QualityTablesFormatter::CountStatistic, _polarizationCount); qualityData.InitializeEmptyStatistic( dimension, QualityTablesFormatter::SumStatistic, _polarizationCount); qualityData.InitializeEmptyStatistic( dimension, QualityTablesFormatter::SumP2Statistic, _polarizationCount); qualityData.InitializeEmptyStatistic( dimension, QualityTablesFormatter::DCountStatistic, _polarizationCount); qualityData.InitializeEmptyStatistic( dimension, QualityTablesFormatter::DSumStatistic, _polarizationCount); qualityData.InitializeEmptyStatistic( dimension, QualityTablesFormatter::DSumP2Statistic, _polarizationCount); } void saveEachStatistic(StatisticSaver& saver, const DefaultStatistics& stat, const Indices& indices) const { StatisticalValue value(_polarizationCount); for (unsigned p = 0; p < _polarizationCount; ++p) value.SetValue(p, std::complex(stat.rfiCount[p], 0.0f)); saver.Save(value, indices.kindRFICount); for (unsigned p = 0; p < _polarizationCount; ++p) value.SetValue(p, std::complex(stat.count[p], 0.0f)); saver.Save(value, indices.kindCount); for (unsigned p = 0; p < _polarizationCount; ++p) value.SetValue(p, stat.Sum(p)); saver.Save(value, indices.kindSum); for (unsigned p = 0; p < _polarizationCount; ++p) value.SetValue(p, stat.SumP2(p)); saver.Save(value, indices.kindSumP2); for (unsigned p = 0; p < _polarizationCount; ++p) value.SetValue(p, std::complex(stat.dCount[p], 0.0f)); saver.Save(value, indices.kindDCount); for (unsigned p = 0; p < _polarizationCount; ++p) value.SetValue(p, stat.DSum(p)); saver.Save(value, indices.kindDSum); for (unsigned p = 0; p < _polarizationCount; ++p) value.SetValue(p, stat.DSumP2(p)); saver.Save(value, indices.kindDSumP2); } void saveTime(QualityTablesFormatter& qd) const; void saveFrequency(QualityTablesFormatter& qd) const; void saveBaseline(QualityTablesFormatter& qd) const; DefaultStatistics& getTimeStatistic(double time, double centralFrequency) { // We use find() to see if the value exists, and only use insert() when it // does not, because insert is slow (because a "Statistic" needs to be // created). Holds for both frequency and time maps. std::map::iterator i = _timeStatistics.find(centralFrequency); if (i == _timeStatistics.end()) { i = _timeStatistics .insert(std::pair(centralFrequency, DoubleStatMap())) .first; } DoubleStatMap& selectedTimeStatistic = i->second; return getDoubleStatMapStatistic(selectedTimeStatistic, time); } DefaultStatistics& getFrequencyStatistic(double frequency) { return getDoubleStatMapStatistic(_frequencyStatistics, frequency); } DefaultStatistics& getDoubleStatMapStatistic(DoubleStatMap& map, double key) { // Use insert() only when not exist, as it is slower then find because a // Statistic is created. DoubleStatMap::iterator i = map.find(key); if (i == map.end()) { i = map.insert(std::pair( key, DefaultStatistics(_polarizationCount))) .first; } return i->second; } DefaultStatistics& getBaselineStatistic(unsigned antenna1, unsigned antenna2, double centralFrequency) { std::map::iterator i = _baselineStatistics.find(centralFrequency); if (i == _baselineStatistics.end()) { i = _baselineStatistics .insert(std::pair( centralFrequency, BaselineStatisticsMap(_polarizationCount))) .first; } BaselineStatisticsMap& selectedBaselineStatistic = i->second; return selectedBaselineStatistic.GetStatistics(antenna1, antenna2); } template void assignOrAdd(T& value, const T otherValue) { if (PerformAdd) value += otherValue; else value = otherValue; } template void assignStatistic(DefaultStatistics& destination, const StatisticalValue& source, QualityTablesFormatter::StatisticKind kind) { for (unsigned p = 0; p < _polarizationCount; ++p) { switch (kind) { case QualityTablesFormatter::RFICountStatistic: assignOrAdd(destination.rfiCount[p], (long unsigned)source.Value(p).real()); break; case QualityTablesFormatter::CountStatistic: assignOrAdd(destination.count[p], (long unsigned)source.Value(p).real()); break; case QualityTablesFormatter::SumStatistic: assignOrAdd( destination.sum[p], std::complex(source.Value(p).real(), source.Value(p).imag())); break; case QualityTablesFormatter::SumP2Statistic: assignOrAdd( destination.sumP2[p], std::complex(source.Value(p).real(), source.Value(p).imag())); break; case QualityTablesFormatter::DCountStatistic: assignOrAdd(destination.dCount[p], (long unsigned)source.Value(p).real()); break; case QualityTablesFormatter::DSumStatistic: assignOrAdd( destination.dSum[p], std::complex(source.Value(p).real(), source.Value(p).imag())); break; case QualityTablesFormatter::DSumP2Statistic: assignOrAdd( destination.dSumP2[p], std::complex(source.Value(p).real(), source.Value(p).imag())); break; default: break; } } } void forEachDefaultStatistic(QualityTablesFormatter& qd, void (StatisticsCollection::*functionName)( QualityTablesFormatter&, QualityTablesFormatter::StatisticKind)) { (this->*functionName)(qd, QualityTablesFormatter::CountStatistic); (this->*functionName)(qd, QualityTablesFormatter::SumStatistic); (this->*functionName)(qd, QualityTablesFormatter::SumP2Statistic); (this->*functionName)(qd, QualityTablesFormatter::DCountStatistic); (this->*functionName)(qd, QualityTablesFormatter::DSumStatistic); (this->*functionName)(qd, QualityTablesFormatter::DSumP2Statistic); (this->*functionName)(qd, QualityTablesFormatter::RFICountStatistic); } template void loadSingleTimeStatistic(QualityTablesFormatter& qd, QualityTablesFormatter::StatisticKind kind) { std::vector< std::pair> values; unsigned kindIndex = qd.QueryKindIndex(kind); qd.QueryTimeStatistic(kindIndex, values); for (std::vector>::const_iterator i = values.begin(); i != values.end(); ++i) { const QualityTablesFormatter::TimePosition& position = i->first; const StatisticalValue& statValue = i->second; DefaultStatistics& stat = getTimeStatistic(position.time, position.frequency); assignStatistic(stat, statValue, kind); } } template void loadTime(QualityTablesFormatter& qd) { forEachDefaultStatistic( qd, &StatisticsCollection::loadSingleTimeStatistic); } template void loadSingleFrequencyStatistic( QualityTablesFormatter& qd, QualityTablesFormatter::StatisticKind kind) { std::vector< std::pair> values; unsigned kindIndex = qd.QueryKindIndex(kind); qd.QueryFrequencyStatistic(kindIndex, values); for (std::vector>::const_iterator i = values.begin(); i != values.end(); ++i) { const QualityTablesFormatter::FrequencyPosition& position = i->first; const StatisticalValue& statValue = i->second; DefaultStatistics& stat = getFrequencyStatistic(position.frequency); assignStatistic(stat, statValue, kind); } } template void loadFrequency(QualityTablesFormatter& qd) { forEachDefaultStatistic( qd, &StatisticsCollection::loadSingleFrequencyStatistic); } template void loadSingleBaselineStatistic(QualityTablesFormatter& qd, QualityTablesFormatter::StatisticKind kind) { std::vector< std::pair> values; unsigned kindIndex = qd.QueryKindIndex(kind); qd.QueryBaselineStatistic(kindIndex, values); for (std::vector>::const_iterator i = values.begin(); i != values.end(); ++i) { const QualityTablesFormatter::BaselinePosition& position = i->first; const StatisticalValue& statValue = i->second; DefaultStatistics& stat = getBaselineStatistic( position.antenna1, position.antenna2, position.frequency); assignStatistic(stat, statValue, kind); } } template void loadBaseline(QualityTablesFormatter& qd) { forEachDefaultStatistic( qd, &StatisticsCollection::loadSingleBaselineStatistic); } double centralFrequency() const { double min = _frequencyStatistics.begin()->first; double max = _frequencyStatistics.rbegin()->first; return (min + max) / 2.0; } DefaultStatistics getGlobalStatistics(const DoubleStatMap& statMap) const { DefaultStatistics global(_polarizationCount); for (DoubleStatMap::const_iterator i = statMap.begin(); i != statMap.end(); ++i) { const DefaultStatistics& stat = i->second; global += stat; } return global; } DefaultStatistics getGlobalStatistics( const std::map& statMap) const { DefaultStatistics global(_polarizationCount); for (std::map::const_iterator i = statMap.begin(); i != statMap.end(); ++i) { const DefaultStatistics& stat = getGlobalStatistics(i->second); global += stat; } return global; } template DefaultStatistics getGlobalBaselineStatistics() const { DefaultStatistics global(_polarizationCount); for (std::map::const_iterator f = _baselineStatistics.begin(); f != _baselineStatistics.end(); ++f) { const BaselineStatisticsMap& map = f->second; const std::vector> baselines = map.BaselineList(); for (std::vector>::const_iterator i = baselines.begin(); i != baselines.end(); ++i) { const unsigned antenna1 = i->first, antenna2 = i->second; if (((antenna1 == antenna2) && AutoCorrelations) || ((antenna1 != antenna2) && (!AutoCorrelations))) { const DefaultStatistics& stat = map.GetStatistics(antenna1, antenna2); global += stat; } } } return global; } DefaultStatistics getFrequencyRangeStatistics(double startFrequency, double endFrequency) const { DefaultStatistics rangeStats(_polarizationCount); for (DoubleStatMap::const_iterator f = _frequencyStatistics.begin(); f != _frequencyStatistics.end(); ++f) { const double frequency = f->first; const DefaultStatistics& stat = f->second; if (frequency >= startFrequency && frequency < endFrequency) rangeStats += stat; } return rangeStats; } void serializeTime(std::ostream& stream) const { SerializeToUInt64(stream, _timeStatistics.size()); for (std::map::const_iterator i = _timeStatistics.begin(); i != _timeStatistics.end(); ++i) { const double frequency = i->first; const DoubleStatMap& map = i->second; SerializeToDouble(stream, frequency); serializeDoubleStatMap(stream, map); } } void unserializeTime(std::istream& stream) { _timeStatistics.clear(); size_t count = (size_t)UnserializeUInt64(stream); std::map::iterator insertPos = _timeStatistics.begin(); for (size_t i = 0; i < count; ++i) { double frequency = UnserializeDouble(stream); insertPos = _timeStatistics.insert( insertPos, std::pair(frequency, DoubleStatMap())); unserializeDoubleStatMap(stream, insertPos->second); } } void serializeFrequency(std::ostream& stream) const { serializeDoubleStatMap(stream, _frequencyStatistics); } void unserializeFrequency(std::istream& stream) { _frequencyStatistics.clear(); unserializeDoubleStatMap(stream, _frequencyStatistics); } void serializeBaselines(std::ostream& stream) const { SerializeToUInt64(stream, _baselineStatistics.size()); for (std::map::const_iterator i = _baselineStatistics.begin(); i != _baselineStatistics.end(); ++i) { const double frequency = i->first; const BaselineStatisticsMap& map = i->second; SerializeToDouble(stream, frequency); map.Serialize(stream); } } void unserializeBaselines(std::istream& stream) { _baselineStatistics.clear(); size_t count = (size_t)UnserializeUInt64(stream); std::map::iterator insertPos = _baselineStatistics.begin(); for (size_t i = 0; i < count; ++i) { double frequency = UnserializeDouble(stream); insertPos = _baselineStatistics.insert( insertPos, std::pair( frequency, BaselineStatisticsMap(_polarizationCount))); insertPos->second.Unserialize(stream); } } void serializeDoubleStatMap(std::ostream& stream, const DoubleStatMap& statMap) const { uint64_t statCount = statMap.size(); stream.write(reinterpret_cast(&statCount), sizeof(statCount)); for (DoubleStatMap::const_iterator i = statMap.begin(); i != statMap.end(); ++i) { const double& key = i->first; const DefaultStatistics& stat = i->second; stream.write(reinterpret_cast(&key), sizeof(key)); stat.Serialize(stream); } } void unserializeDoubleStatMap(std::istream& stream, DoubleStatMap& statMap) const { size_t count = (size_t)UnserializeUInt64(stream); std::map::iterator insertPos = statMap.begin(); for (size_t i = 0; i < count; ++i) { double key = UnserializeDouble(stream); insertPos = statMap.insert( insertPos, std::pair( key, DefaultStatistics(_polarizationCount))); insertPos->second.Unserialize(stream); } } void addTime(const StatisticsCollection& collection) { for (std::map::const_iterator i = collection._timeStatistics.begin(); i != collection._timeStatistics.end(); ++i) { const double frequency = i->first; const DoubleStatMap& map = i->second; for (DoubleStatMap::const_iterator j = map.begin(); j != map.end(); ++j) { const double time = j->first; const DefaultStatistics& stat = j->second; getTimeStatistic(time, frequency) += stat; } } } void addFrequency(const StatisticsCollection& collection) { for (DoubleStatMap::const_iterator j = collection._frequencyStatistics.begin(); j != collection._frequencyStatistics.end(); ++j) { const double frequency = j->first; const DefaultStatistics& stat = j->second; getFrequencyStatistic(frequency) += stat; } } void addBaseline(const StatisticsCollection& collection) { for (std::map::const_iterator i = collection._baselineStatistics.begin(); i != collection._baselineStatistics.end(); ++i) { const double frequency = i->first; const BaselineStatisticsMap& map = i->second; std::vector> baselines = map.BaselineList(); for (std::vector>::const_iterator j = baselines.begin(); j != baselines.end(); ++j) { const unsigned antenna1 = j->first; const unsigned antenna2 = j->second; const DefaultStatistics& stat = map.GetStatistics(antenna1, antenna2); getBaselineStatistic(antenna1, antenna2, frequency) += stat; } } } void addToDoubleStatMap(DoubleStatMap& dest, const DoubleStatMap& source) { for (DoubleStatMap::const_iterator i = source.begin(); i != source.end(); ++i) { double key = i->first; const DefaultStatistics& sourceStats = i->second; getDoubleStatMapStatistic(dest, key) += sourceStats; } } void lowerResolution(DoubleStatMap& map, size_t maxSteps) const; static void regrid(const DoubleStatMap& referenceMap, DoubleStatMap& regridMap) { DoubleStatMap newMap; for (DoubleStatMap::const_iterator i = regridMap.begin(); i != regridMap.end(); ++i) { double key = i->first; // find the key in the reference map that is closest to this key, if it is // within range DoubleStatMap::const_iterator bound = referenceMap.lower_bound(key); if (bound != referenceMap.end()) { double rightKey = bound->first; if (bound != referenceMap.begin()) { --bound; double leftKey = bound->first; if (key - rightKey < leftKey - key) key = rightKey; else key = leftKey; } } newMap.insert(std::pair(key, i->second)); } regridMap = newMap; } static void addAntennaStatistic( unsigned antIndex, const DefaultStatistics& stats, std::map& antStatistics) { std::map::iterator iter = antStatistics.find(antIndex); if (iter == antStatistics.end()) antStatistics.insert(std::make_pair(antIndex, stats)); else iter->second += stats; } std::map _timeStatistics; DoubleStatMap _frequencyStatistics; std::map _baselineStatistics; std::map> _bands; std::map _centralFrequencies; std::map> _bandFrequencies; unsigned _polarizationCount; BaselineStatisticsMap _emptyBaselineStatisticsMap; }; #endif aoflagger-v3.5.1/quality/statisticalvalue.h0000664000175000017500000000312214752462134017207 0ustar oleole#ifndef MSIO_STATISTICAL_VALUE_H #define MSIO_STATISTICAL_VALUE_H #include class StatisticalValue { public: explicit StatisticalValue(unsigned _polarizationCount) : _polarizationCount(_polarizationCount), _kindIndex(0), _values(new std::complex[_polarizationCount]) {} StatisticalValue(const StatisticalValue& source) : _polarizationCount(source._polarizationCount), _kindIndex(source._kindIndex), _values(new std::complex[source._polarizationCount]) { for (unsigned i = 0; i < _polarizationCount; ++i) _values[i] = source._values[i]; } ~StatisticalValue() { delete[] _values; } StatisticalValue& operator=(const StatisticalValue& source) { if (_polarizationCount != source._polarizationCount) { _polarizationCount = source._polarizationCount; delete[] _values; _values = new std::complex[_polarizationCount]; } _kindIndex = source._kindIndex; for (unsigned i = 0; i < _polarizationCount; ++i) _values[i] = source._values[i]; return *this; } unsigned PolarizationCount() const { return _polarizationCount; } unsigned KindIndex() const { return _kindIndex; } void SetKindIndex(unsigned kindIndex) { _kindIndex = kindIndex; } std::complex Value(unsigned polarizationIndex) const { return _values[polarizationIndex]; } void SetValue(unsigned polarizationIndex, std::complex newValue) { _values[polarizationIndex] = newValue; } private: unsigned _polarizationCount; unsigned _kindIndex; std::complex* _values; }; #endif aoflagger-v3.5.1/quality/rayleighfitter.cpp0000664000175000017500000001532614752462134017214 0ustar oleole#include "rayleighfitter.h" #include #include #include #ifdef HAVE_GSL #include #include #include static int fit_f(const gsl_vector* xvec, void* data, gsl_vector* f) { const double sigma = gsl_vector_get(xvec, 0); const double n = gsl_vector_get(xvec, 1); size_t t = 0; const RayleighFitter& fitter = *static_cast(data); const LogHistogram& hist = *fitter._hist; const double minVal = fitter._minVal; const double maxVal = fitter._maxVal; for (LogHistogram::const_iterator i = hist.begin(); i != hist.end(); ++i) { const double x = i.value(); if (x >= minVal && x < maxVal && std::isfinite(x)) { const double val = i.normalizedCount(); // const double logval = log(val); // const double weight = logval; const double sigmaP2 = sigma * sigma; const double Yi = x * exp(-(x * x) / (2 * sigmaP2)) * n / sigmaP2; if (fitter.FitLogarithmic()) gsl_vector_set(f, t, log(Yi) - log(val)); else gsl_vector_set(f, t, (Yi - val)); ++t; } } return GSL_SUCCESS; } int fit_df(const gsl_vector* xvec, void* data, gsl_matrix* J) { const double sigma = gsl_vector_get(xvec, 0); const double n = gsl_vector_get(xvec, 1); size_t t = 0; const RayleighFitter& fitter = *static_cast(data); const LogHistogram& hist = *fitter._hist; const double minVal = fitter._minVal; const double maxVal = fitter._maxVal; const double sigmaP2 = sigma * sigma; const double sigmaP3 = sigma * sigma * sigma; for (LogHistogram::const_iterator i = hist.begin(); i != hist.end(); ++i) { const double x = i.value(); if (x >= minVal && x < maxVal && std::isfinite(x)) { // const double val = i.normalizedCount(); // const double weight = log(val); double dfdsigma, dfdn; if (fitter.FitLogarithmic()) { dfdsigma = (x * x - 2.0 * sigmaP2) / sigmaP3; dfdn = 1.0 / n; } else { dfdsigma = -n * 2.0 * x * x * x / (sigmaP3 * sigmaP3) * exp(-x * x / (2.0 * sigmaP2)); dfdn = x * exp(-(x * x) / (2 * sigmaP2)) / sigmaP2; } // diff to sigma gsl_matrix_set(J, t, 0, dfdsigma); // diff to n gsl_matrix_set(J, t, 1, dfdn); ++t; } } return GSL_SUCCESS; } int fit_fdf(const gsl_vector* x, void* data, gsl_vector* f, gsl_matrix* J) { fit_f(x, data, f); fit_df(x, data, J); return GSL_SUCCESS; } void print_state(size_t iter, gsl_multifit_fdfsolver* s) { const double sigma = gsl_vector_get(s->x, 0); const double N = gsl_vector_get(s->x, 1); std::cout << "iteration " << iter << ", sigma=" << sigma << ", N=" << N << "\n"; } void RayleighFitter::Fit(double minVal, double maxVal, const LogHistogram& hist, double& sigma, double& n) { unsigned int iter = 0; const size_t nVars = 2; _hist = &hist; if (minVal > 0) _minVal = minVal; else _minVal = hist.MinPositiveAmplitude(); _maxVal = maxVal; if (sigma < minVal) sigma = minVal; size_t nData = 0; for (LogHistogram::iterator i = hist.begin(); i != hist.end(); ++i) { const double val = i.value(); if (val >= minVal && val < maxVal && std::isfinite(val)) ++nData; } std::cout << "ndata=" << nData << "\n"; double x_init[nVars] = {sigma, n}; const gsl_vector_view x = gsl_vector_view_array(x_init, nVars); gsl_multifit_function_fdf f; f.f = &fit_f; f.df = &fit_df; f.fdf = &fit_fdf; f.n = nData; f.p = nVars; f.params = this; const gsl_multifit_fdfsolver_type* T = gsl_multifit_fdfsolver_lmsder; gsl_multifit_fdfsolver* s = gsl_multifit_fdfsolver_alloc(T, nData, nVars); gsl_multifit_fdfsolver_set(s, &f, &x.vector); print_state(iter, s); int status; do { iter++; status = gsl_multifit_fdfsolver_iterate(s); std::cout << "status = " << gsl_strerror(status) << "\n"; print_state(iter, s); if (status) break; status = gsl_multifit_test_delta(s->dx, s->x, 1e-7, 1e-3); } while (status == GSL_CONTINUE && iter < 500); std::cout << "status = " << gsl_strerror(status) << "\n"; print_state(iter, s); sigma = fabs(gsl_vector_get(s->x, 0)); n = fabs(gsl_vector_get(s->x, 1)); gsl_multifit_fdfsolver_free(s); } #else // No gsl... void RayleighFitter::Fit(double minVal, double maxVal, const LogHistogram& hist, double& sigma, double& n) { sigma = 1.0; n = 1.0; } #endif double RayleighFitter::SigmaEstimate(const LogHistogram& hist) { return hist.AmplitudeWithMaxNormalizedCount(); } double RayleighFitter::SigmaEstimate(const LogHistogram& hist, double rangeStart, double rangeEnd) { double maxAmplitude = 0.0, maxNormalizedCount = boost::numeric::bounds::lowest(); for (LogHistogram::const_iterator i = hist.begin(); i != hist.end(); ++i) { if (i.value() > rangeStart && i.value() < rangeEnd && std::isfinite(i.value())) { if (std::isfinite(i.normalizedCount())) { if (i.normalizedCount() > maxNormalizedCount) { maxAmplitude = i.value(); maxNormalizedCount = i.normalizedCount(); } } } } return maxAmplitude; } void RayleighFitter::FindFitRangeUnderRFIContamination( double minPositiveAmplitude, double sigmaEstimate, double& minValue, double& maxValue) { minValue = minPositiveAmplitude; maxValue = sigmaEstimate * 1.5; std::cout << "Found range " << minValue << " -- " << maxValue << "\n"; } double RayleighFitter::ErrorOfFit(const LogHistogram& histogram, double rangeStart, double rangeEnd, double sigma, double n) { double sum = 0.0; size_t count = 0; for (LogHistogram::const_iterator i = histogram.begin(); i != histogram.end(); ++i) { const double x = i.value(); if (x >= rangeStart && x < rangeEnd && std::isfinite(x)) { const double val = i.normalizedCount(); const double sigmaP2 = sigma * sigma; const double Yi = x * exp(-(x * x) / (2 * sigmaP2)) * n / sigmaP2; const double error = (Yi - val) * (Yi - val); sum += error; ++count; } } return sum / (double)count; } double RayleighFitter::NEstimate(const LogHistogram& hist, double rangeStart, double rangeEnd) { double rangeSum = 0.0; size_t count = 0; for (LogHistogram::const_iterator i = hist.begin(); i != hist.end(); ++i) { if (i.value() > rangeStart && i.value() < rangeEnd && std::isfinite(i.value())) { if (std::isfinite(i.normalizedCount())) { rangeSum += i.normalizedCount(); ++count; } } } return rangeSum / (count * 10.0); } aoflagger-v3.5.1/quality/baselinestatisticsmap.h0000664000175000017500000001303014752462134020220 0ustar oleole#ifndef BASELINESTATISTICSMAP_H #define BASELINESTATISTICSMAP_H #include #include #include #include "../util/serializable.h" #include "defaultstatistics.h" class BaselineStatisticsMap : public Serializable { public: explicit BaselineStatisticsMap(unsigned polarizationCount) : _polarizationCount(polarizationCount) {} void operator+=(const BaselineStatisticsMap& other) { for (OuterMap::const_iterator i = other._map.begin(); i != other._map.end(); ++i) { unsigned antenna1 = i->first; const InnerMap& innerMap = i->second; for (InnerMap::const_iterator j = innerMap.begin(); j != innerMap.end(); ++j) { unsigned antenna2 = j->first; const DefaultStatistics& stat = j->second; GetStatistics(antenna1, antenna2) += stat; } } } DefaultStatistics& GetStatistics(unsigned antenna1, unsigned antenna2) { OuterMap::iterator antenna1Map = _map.insert(OuterPair(antenna1, InnerMap())).first; InnerMap& innerMap = antenna1Map->second; InnerMap::iterator antenna2Value = innerMap.find(antenna2); DefaultStatistics* statistics; if (antenna2Value == innerMap.end()) { // The baseline does not exist yet, create empty statistics. statistics = &(innerMap .insert(InnerPair( antenna2, DefaultStatistics(_polarizationCount))) .first->second); } else { statistics = &antenna2Value->second; } return *statistics; } const DefaultStatistics& GetStatistics(unsigned antenna1, unsigned antenna2) const { OuterMap::const_iterator antenna1Map = _map.find(antenna1); if (antenna1Map == _map.end()) { throw std::runtime_error( "BaselineStatisticsMap::GetStatistics() : Requested unavailable " "baseline"); } else { const InnerMap& innerMap = antenna1Map->second; InnerMap::const_iterator antenna2Value = innerMap.find(antenna2); if (antenna2Value == innerMap.end()) { throw std::runtime_error( "BaselineStatisticsMap::GetStatistics() : Requested unavailable " "baseline"); } else { return antenna2Value->second; } } } std::vector> BaselineList() const { std::vector> list; for (OuterMap::const_iterator outerIter = _map.begin(); outerIter != _map.end(); ++outerIter) { const unsigned antenna1 = outerIter->first; const InnerMap& innerMap = outerIter->second; for (InnerMap::const_iterator innerIter = innerMap.begin(); innerIter != innerMap.end(); ++innerIter) { const unsigned antenna2 = innerIter->first; list.push_back(std::pair(antenna1, antenna2)); } } return list; } unsigned AntennaCount() const { if (_map.empty()) return 0; unsigned highestIndex = _map.rbegin()->first; for (OuterMap::const_iterator outerIter = _map.begin(); outerIter != _map.end(); ++outerIter) { const InnerMap& innerMap = outerIter->second; if (highestIndex < innerMap.rbegin()->first) highestIndex = innerMap.rbegin()->first; } return highestIndex + 1; } void Clear() { _map.clear(); } unsigned PolarizationCount() const { return _polarizationCount; } virtual void Serialize(std::ostream& stream) const final override { SerializeToUInt32(stream, _polarizationCount); serializeOuterMap(stream, _map); } virtual void Unserialize(std::istream& stream) final override { _map.clear(); _polarizationCount = UnserializeUInt32(stream); unserializeOuterMap(stream, _map); } private: typedef std::map InnerMap; typedef std::pair InnerPair; typedef std::map OuterMap; typedef std::pair OuterPair; OuterMap _map; unsigned _polarizationCount; void serializeOuterMap(std::ostream& stream, const OuterMap& map) const { SerializeToUInt32(stream, map.size()); for (OuterMap::const_iterator i = map.begin(); i != map.end(); ++i) { unsigned antenna1 = i->first; SerializeToUInt32(stream, antenna1); const InnerMap& innerMap = i->second; serializeInnerMap(stream, innerMap); } } void unserializeOuterMap(std::istream& stream, OuterMap& map) const { size_t size = UnserializeUInt32(stream); for (size_t j = 0; j < size; ++j) { unsigned antenna1 = UnserializeUInt32(stream); OuterMap::iterator i = map.insert(std::pair(antenna1, InnerMap())).first; unserializeInnerMap(stream, i->second); } } void serializeInnerMap(std::ostream& stream, const InnerMap& map) const { SerializeToUInt32(stream, map.size()); for (InnerMap::const_iterator i = map.begin(); i != map.end(); ++i) { unsigned antenna2 = i->first; SerializeToUInt32(stream, antenna2); const DefaultStatistics& statistics = i->second; statistics.Serialize(stream); } } void unserializeInnerMap(std::istream& stream, InnerMap& map) const { size_t size = UnserializeUInt32(stream); for (size_t j = 0; j < size; ++j) { unsigned antenna2 = UnserializeUInt32(stream); InnerMap::iterator i = map.insert(std::pair( antenna2, DefaultStatistics(_polarizationCount))) .first; i->second.Unserialize(stream); } } }; #endif aoflagger-v3.5.1/quality/histogramcollection.cpp0000664000175000017500000001014114752462134020231 0ustar oleole#include "histogramcollection.h" #include "histogramtablesformatter.h" void HistogramCollection::Save(HistogramTablesFormatter& histogramTables) { histogramTables.InitializeEmptyTables(); for (size_t p = 0; p < _polarizationCount; ++p) { LogHistogram totalHistogram; GetTotalHistogramForCrossCorrelations(p, totalHistogram); const unsigned totalIndex = histogramTables.StoreOrQueryTypeIndex( HistogramTablesFormatter::TotalHistogram, p); for (LogHistogram::iterator i = totalHistogram.begin(); i != totalHistogram.end(); ++i) { histogramTables.StoreValue(totalIndex, i.binStart(), i.binEnd(), i.unnormalizedCount()); } LogHistogram rfiHistogram; GetRFIHistogramForCrossCorrelations(p, rfiHistogram); const unsigned rfiIndex = histogramTables.StoreOrQueryTypeIndex( HistogramTablesFormatter::RFIHistogram, p); for (LogHistogram::iterator i = rfiHistogram.begin(); i != rfiHistogram.end(); ++i) { histogramTables.StoreValue(rfiIndex, i.binStart(), i.binEnd(), i.unnormalizedCount()); } } } void HistogramCollection::Load(HistogramTablesFormatter& histogramTables) { Clear(); for (unsigned p = 0; p < _polarizationCount; ++p) { const unsigned totalHistogramIndex = histogramTables.QueryTypeIndex( HistogramTablesFormatter::TotalHistogram, p); std::vector totalHistogram; histogramTables.QueryHistogram(totalHistogramIndex, totalHistogram); GetTotalHistogram(0, 1, p).SetData(totalHistogram); const unsigned rfiHistogramIndex = histogramTables.QueryTypeIndex( HistogramTablesFormatter::RFIHistogram, p); std::vector rfiHistogram; histogramTables.QueryHistogram(rfiHistogramIndex, rfiHistogram); GetRFIHistogram(0, 1, p).SetData(rfiHistogram); } } void HistogramCollection::Add(const unsigned antenna1, const unsigned antenna2, const unsigned polarization, Image2DCPtr image, Mask2DCPtr flagMask) { LogHistogram& totalHistogram = GetTotalHistogram(antenna1, antenna2, polarization); LogHistogram& rfiHistogram = GetRFIHistogram(antenna1, antenna2, polarization); for (size_t y = 0; y < image->Height(); ++y) { for (size_t x = 0; x < image->Width(); ++x) { const double amplitude = image->Value(x, y); totalHistogram.Add(amplitude); if (flagMask->Value(x, y)) rfiHistogram.Add(amplitude); } } } void HistogramCollection::Add(const unsigned antenna1, const unsigned antenna2, const unsigned polarization, Image2DCPtr image, Mask2DCPtr flagMask, Mask2DCPtr correlatorMask) { LogHistogram& totalHistogram = GetTotalHistogram(antenna1, antenna2, polarization); LogHistogram& rfiHistogram = GetRFIHistogram(antenna1, antenna2, polarization); for (size_t y = 0; y < image->Height(); ++y) { for (size_t x = 0; x < image->Width(); ++x) { if (!correlatorMask->Value(x, y)) { const double amplitude = image->Value(x, y); totalHistogram.Add(amplitude); if (flagMask->Value(x, y)) rfiHistogram.Add(amplitude); } } } } void HistogramCollection::Add(const unsigned antenna1, const unsigned antenna2, const unsigned polarization, Image2DCPtr real, Image2DCPtr imaginary, Mask2DCPtr flagMask, Mask2DCPtr correlatorMask) { LogHistogram& totalHistogram = GetTotalHistogram(antenna1, antenna2, polarization); LogHistogram& rfiHistogram = GetRFIHistogram(antenna1, antenna2, polarization); for (size_t y = 0; y < real->Height(); ++y) { for (size_t x = 0; x < real->Width(); ++x) { if (!correlatorMask->Value(x, y)) { const double r = real->Value(x, y), i = imaginary->Value(x, y); const double amplitude = sqrt(r * r + i * i); totalHistogram.Add(amplitude); if (flagMask->Value(x, y)) rfiHistogram.Add(amplitude); } } } } aoflagger-v3.5.1/quality/operations.h0000664000175000017500000000440614752462134016017 0ustar oleole#ifndef AOFLAGGER_QUALITY_OPERATIONS_H_ #define AOFLAGGER_QUALITY_OPERATIONS_H_ #include #include #include #include "collector.h" namespace quality { void ListStatistics(); void CollectStatistics(const std::string& filename, Collector::CollectingMode mode, size_t flaggedTimesteps, std::set&& flaggedAntennae, const char* dataColumnName, size_t intervalStart, size_t intervalEnd); void CollectHistograms(const std::string& filename, HistogramCollection& histogramCollection, size_t flaggedTimesteps, std::set&& flaggedAntennae, const char* dataColumnName); void CombineStatistics(const std::string& result_filename, const std::vector& input_filenames); void RemoveStatistics(const std::string& filename); void RemoveHistogram(const std::string& filename); void PrintSummary(const std::vector& filenames); void PrintRfiSummary(const std::vector& filenames); void PrintGlobalStatistic(const std::string& kindName, const std::vector& filenames); void PrintPerAntennaStatistics(const std::string& kindName, const std::vector& filenames); void PrintPerBaselineStatistics(const std::string& kindName, const std::vector& filenames); void PrintPerTimeStatistics(const std::string& kindName, const std::vector& filenames); void PrintPerFrequencyStatistics(const std::string& kindName, const std::vector& filenames, std::optional downsample); void PrintFrequencyRangeStatistic(const std::string& kindName, const std::vector& filenames, double startFreqMHz, double endFreqMHz); void PrintRfiSlope(const std::string& filename); void PrintRfiSlopePerBaseline(const std::string& filename, const char* dataColumnName); } // namespace quality #endif aoflagger-v3.5.1/quality/qualitytablesformatter.h0000664000175000017500000002561514752462134020450 0ustar oleole#ifndef MSIO_QUALITY_DATA_H #define MSIO_QUALITY_DATA_H #include #include #define QUALITY_TABLES_VERSION 1 #define QUALITY_TABLES_VERSION_STR "1" class StatisticalValue; class QualityTablesFormatter { public: enum StatisticKind { CountStatistic, SumStatistic, MeanStatistic, RFICountStatistic, RFISumStatistic, RFIMeanStatistic, RFIRatioStatistic, RFIPercentageStatistic, FlaggedCountStatistic, FlaggedRatioStatistic, SumP2Statistic, SumP3Statistic, SumP4Statistic, VarianceStatistic, VarianceOfVarianceStatistic, StandardDeviationStatistic, SkewnessStatistic, KurtosisStatistic, SignalToNoiseStatistic, DSumStatistic, DMeanStatistic, DSumP2Statistic, DSumP3Statistic, DSumP4Statistic, DVarianceStatistic, DVarianceOfVarianceStatistic, DStandardDeviationStatistic, DCountStatistic, BadSolutionCountStatistic, CorrectCountStatistic, CorrectedMeanStatistic, CorrectedSumP2Statistic, CorrectedDCountStatistic, CorrectedDMeanStatistic, CorrectedDSumP2Statistic, FTSumStatistic, FTSumP2Statistic, EndPlaceHolderStatistic }; enum StatisticDimension { TimeDimension, FrequencyDimension, BaselineDimension, BaselineTimeDimension }; enum QualityTable { KindNameTable, TimeStatisticTable, FrequencyStatisticTable, BaselineStatisticTable, BaselineTimeStatisticTable }; struct TimePosition { double time; double frequency; }; struct FrequencyPosition { double frequency; }; struct BaselinePosition { unsigned antenna1; unsigned antenna2; double frequency; }; struct BaselineTimePosition { double time; unsigned antenna1; unsigned antenna2; double frequency; }; explicit QualityTablesFormatter(const std::string& measurementSetName) : _measurementSet(), _measurementSetName(measurementSetName), _kindNameTable(), _timeTable(), _frequencyTable(), _baselineTable(), _baselineTimeTable() {} ~QualityTablesFormatter() { Close(); } void Close() { _kindNameTable.reset(); _timeTable.reset(); _frequencyTable.reset(); _baselineTable.reset(); _baselineTimeTable.reset(); _measurementSet.reset(); } bool TableExists(enum QualityTable table) const { return _measurementSet->isReadable(TableToFilename(table)); } static const std::string& KindToName(const enum StatisticKind kind) { return _kindToNameTable[(int)kind]; } static enum StatisticKind NameToKind(const std::string& kindName); static const std::string& TableToName(const enum QualityTable table) { return _tableToNameTable[(int)table]; } const std::string TableToFilename(const enum QualityTable table) const { return _measurementSetName + '/' + TableToName(table); } enum QualityTable DimensionToTable( const enum StatisticDimension dimension) const { return _dimensionToTableTable[(int)dimension]; } bool IsStatisticAvailable(enum StatisticDimension dimension, enum StatisticKind kind); void InitializeEmptyStatistic(enum StatisticDimension dimension, enum StatisticKind kind, unsigned polarizationCount); void InitializeEmptyTable(enum QualityTable table, unsigned polarizationCount); void RemoveTable(enum QualityTable table); void RemoveAllQualityTables() { RemoveTable(BaselineTimeStatisticTable); RemoveTable(BaselineStatisticTable); RemoveTable(FrequencyStatisticTable); RemoveTable(TimeStatisticTable); RemoveTable(KindNameTable); } unsigned StoreKindName(enum StatisticKind kind) { return StoreKindName(KindToName(kind)); } unsigned StoreKindName(const std::string& name); void StoreTimeValue(double time, double frequency, const StatisticalValue& value); void StoreFrequencyValue(double frequency, const StatisticalValue& value); void StoreBaselineValue(unsigned antenna1, unsigned antenna2, double frequency, const StatisticalValue& value); void StoreBaselineTimeValue(unsigned antenna1, unsigned antenna2, double time, double frequency, const StatisticalValue& value); unsigned QueryKindIndex(enum StatisticKind kind); bool QueryKindIndex(enum StatisticKind kind, unsigned& destKindIndex); unsigned StoreOrQueryKindIndex(enum StatisticKind kind) { unsigned kindIndex; if (QueryKindIndex(kind, kindIndex)) return kindIndex; else return StoreKindName(kind); } unsigned QueryStatisticEntryCount(enum StatisticDimension dimension, unsigned kindIndex); void QueryTimeStatistic( unsigned kindIndex, std::vector>& entries); void QueryFrequencyStatistic( unsigned kindIndex, std::vector>& entries); void QueryBaselineStatistic( unsigned kindIndex, std::vector>& entries); void QueryBaselineTimeStatistic( unsigned kindIndex, std::vector>& entries); unsigned GetPolarizationCount(); private: QualityTablesFormatter(const QualityTablesFormatter&) = delete; // don't allow copies void operator=(const QualityTablesFormatter&) = delete; // don't allow assignment static const std::string _kindToNameTable[]; static const std::string _tableToNameTable[]; static const QualityTable _dimensionToTableTable[]; static const std::string ColumnNameAntenna1; static const std::string ColumnNameAntenna2; static const std::string ColumnNameFrequency; static const std::string ColumnNameKind; static const std::string ColumnNameName; static const std::string ColumnNameTime; static const std::string ColumnNameValue; std::unique_ptr _measurementSet; const std::string _measurementSetName; std::unique_ptr _kindNameTable; std::unique_ptr _timeTable; std::unique_ptr _frequencyTable; std::unique_ptr _baselineTable; std::unique_ptr _baselineTimeTable; bool hasOneEntry(enum QualityTable table, unsigned kindIndex); void removeStatisticFromStatTable(enum QualityTable table, enum StatisticKind kind); void removeKindNameEntry(enum StatisticKind kind); void removeEntries(enum QualityTable table); /** * Add the time column to the table descriptor. Used by create..Table() * methods. It holds "Measure"s of time, which is what casacore defines as a * value including a unit and a reference frame. */ void addTimeColumn(casacore::TableDesc& tableDesc); /** * Add the frequency column to the table descriptor. Used by create..Table() * methods. It holds "Quantum"s of frequency, which is what casacore defines * as a value including a unit (Hertz). */ void addFrequencyColumn(casacore::TableDesc& tableDesc); /** * Add value column to the table descriptor. Used by create..Table() methods. * It consist of an array of statistics, each element holds a polarization. */ void addValueColumn(casacore::TableDesc& tableDesc, unsigned polarizationCount); void createTable(enum QualityTable table, unsigned polarizationCount) { switch (table) { case KindNameTable: createKindNameTable(); break; case TimeStatisticTable: createTimeStatisticTable(polarizationCount); break; case FrequencyStatisticTable: createFrequencyStatisticTable(polarizationCount); break; case BaselineStatisticTable: createBaselineStatisticTable(polarizationCount); break; case BaselineTimeStatisticTable: createBaselineTimeStatisticTable(polarizationCount); break; default: break; } } /** * Will add an empty table to the measurement set named "QUALITY_KIND_NAME" * and initialize its default column. This table can hold a list of quality * statistic types that are referred to in the statistic value tables. */ void createKindNameTable(); /** * Will add an empty table to the measurement set named * "QUALITY_TIME_STATISTIC" and initialize its default column. This table can * hold several statistic kinds per time step. * @param polarizationCount specifies the nr polarizations. This is required * for the shape of the value column. */ void createTimeStatisticTable(unsigned polarizationCount); /** * Will add an empty table to the measurement set named * "QUALITY_FREQUENCY_STATISTIC" and initialize its default column. This table * can hold several statistic kinds per time step. * @param polarizationCount specifies the nr polarizations. This is required * for the shape of the value column. */ void createFrequencyStatisticTable(unsigned polarizationCount); /** * Will add an empty table to the measurement set named * "QUALITY_BASELINE_STATISTIC" and initialize its default column. This table * can hold several statistic kinds per time step. * @param polarizationCount specifies the nr polarizations. This is required * for the shape of the value column. */ void createBaselineStatisticTable(unsigned polarizationCount); void createBaselineTimeStatisticTable(unsigned polarizationCount); unsigned findFreeKindIndex(casacore::Table& kindTable); void openMainTable(bool needWrite); void openTable(QualityTable table, bool needWrite, std::unique_ptr& tablePtr); void openKindNameTable(bool needWrite) { openTable(KindNameTable, needWrite, _kindNameTable); } void openTimeTable(bool needWrite) { openTable(TimeStatisticTable, needWrite, _timeTable); } void openFrequencyTable(bool needWrite) { openTable(FrequencyStatisticTable, needWrite, _frequencyTable); } void openBaselineTable(bool needWrite) { openTable(BaselineStatisticTable, needWrite, _baselineTable); } void openBaselineTimeTable(bool needWrite) { openTable(BaselineTimeStatisticTable, needWrite, _baselineTimeTable); } casacore::Table& getTable(QualityTable table, bool needWrite) { std::unique_ptr* tablePtr = nullptr; switch (table) { case KindNameTable: tablePtr = &_kindNameTable; break; case TimeStatisticTable: tablePtr = &_timeTable; break; case FrequencyStatisticTable: tablePtr = &_frequencyTable; break; case BaselineStatisticTable: tablePtr = &_baselineTable; break; case BaselineTimeStatisticTable: tablePtr = &_baselineTimeTable; break; } openTable(table, needWrite, *tablePtr); return **tablePtr; } }; #endif aoflagger-v3.5.1/quality/combine.h0000664000175000017500000000102114752462134015236 0ustar oleole#ifndef AOFLAGGER_QUALITY_COMBINE_H_ #define AOFLAGGER_QUALITY_COMBINE_H_ #include #include #include "histogramcollection.h" #include "statisticscollection.h" namespace quality { struct FileContents { StatisticsCollection statistics_collection; HistogramCollection histogram_collection; }; /** * Reads and combines a number of quality statistics tables. */ FileContents ReadAndCombine(const std::vector& files, bool verbose); } // namespace quality #endif aoflagger-v3.5.1/quality/defaultstatistics.h0000664000175000017500000001434714752462134017400 0ustar oleole#ifndef QUALITY__DEFAULT_STATISTICS_H #define QUALITY__DEFAULT_STATISTICS_H #include #include #include "../util/serializable.h" class DefaultStatistics : public Serializable { public: explicit DefaultStatistics(unsigned polarizationCount) : _polarizationCount(polarizationCount) { initialize(); for (unsigned p = 0; p < _polarizationCount; ++p) { rfiCount[p] = 0; count[p] = 0; sum[p] = 0.0; sumP2[p] = 0.0; dCount[p] = 0; dSum[p] = 0.0; dSumP2[p] = 0.0; } } ~DefaultStatistics() { destruct(); } DefaultStatistics(const DefaultStatistics& other) : _polarizationCount(other._polarizationCount) { initialize(); for (unsigned p = 0; p < _polarizationCount; ++p) { rfiCount[p] = other.rfiCount[p]; count[p] = other.count[p]; sum[p] = other.sum[p]; sumP2[p] = other.sumP2[p]; dCount[p] = other.dCount[p]; dSum[p] = other.dSum[p]; dSumP2[p] = other.dSumP2[p]; } } DefaultStatistics& operator=(const DefaultStatistics& other) { if (other._polarizationCount != _polarizationCount) { destruct(); _polarizationCount = other._polarizationCount; initialize(); } for (unsigned p = 0; p < _polarizationCount; ++p) { rfiCount[p] = other.rfiCount[p]; count[p] = other.count[p]; sum[p] = other.sum[p]; sumP2[p] = other.sumP2[p]; dCount[p] = other.dCount[p]; dSum[p] = other.dSum[p]; dSumP2[p] = other.dSumP2[p]; } return *this; } DefaultStatistics& operator+=(const DefaultStatistics& other) { for (unsigned p = 0; p < _polarizationCount; ++p) { rfiCount[p] += other.rfiCount[p]; count[p] += other.count[p]; sum[p] += other.sum[p]; sumP2[p] += other.sumP2[p]; dCount[p] += other.dCount[p]; dSum[p] += other.dSum[p]; dSumP2[p] += other.dSumP2[p]; } return *this; } bool operator==(const DefaultStatistics& rhs) const { if (_polarizationCount != rhs._polarizationCount) return false; for (unsigned p = 0; p < _polarizationCount; ++p) { if (rfiCount[p] != rhs.rfiCount[p]) return false; if (count[p] != rhs.count[p]) return false; if (sum[p] != rhs.sum[p]) return false; if (sumP2[p] != rhs.sumP2[p]) return false; if (dCount[p] != rhs.dCount[p]) return false; if (dSum[p] != rhs.dSum[p]) return false; if (dSumP2[p] != rhs.dSumP2[p]) return false; } return true; } bool operator!=(const DefaultStatistics& rhs) const { return !(*this == rhs); } DefaultStatistics ToSinglePolarization() const { if (_polarizationCount == 1) return *this; DefaultStatistics singlePol(1); for (unsigned p = 0; p < _polarizationCount; ++p) { singlePol.rfiCount[0] += rfiCount[p]; singlePol.count[0] += count[p]; singlePol.sum[0] += sum[p]; singlePol.sumP2[0] += sumP2[p]; singlePol.dCount[0] += dCount[p]; singlePol.dSum[0] += dSum[p]; singlePol.dSumP2[0] += dSumP2[p]; } return singlePol; } virtual void Serialize(std::ostream& stream) const final override { SerializeToUInt32(stream, _polarizationCount); for (unsigned p = 0; p < _polarizationCount; ++p) { SerializeToUInt64(stream, rfiCount[p]); SerializeToUInt64(stream, count[p]); SerializeToLDoubleC(stream, sum[p]); SerializeToLDoubleC(stream, sumP2[p]); SerializeToUInt64(stream, dCount[p]); SerializeToLDoubleC(stream, dSum[p]); SerializeToLDoubleC(stream, dSumP2[p]); } } virtual void Unserialize(std::istream& stream) final override { uint32_t pCount = UnserializeUInt32(stream); if (pCount != _polarizationCount) { destruct(); _polarizationCount = pCount; initialize(); } for (unsigned p = 0; p < _polarizationCount; ++p) { rfiCount[p] = UnserializeUInt64(stream); count[p] = UnserializeUInt64(stream); sum[p] = UnserializeLDoubleC(stream); sumP2[p] = UnserializeLDoubleC(stream); dCount[p] = UnserializeUInt64(stream); dSum[p] = UnserializeLDoubleC(stream); dSumP2[p] = UnserializeLDoubleC(stream); } } unsigned PolarizationCount() const { return _polarizationCount; } template std::complex Mean(unsigned polarization) const { return std::complex(sum[polarization].real() / count[polarization], sum[polarization].imag() / count[polarization]); } template std::complex Sum(unsigned polarization) const { return std::complex(sum[polarization].real(), sum[polarization].imag()); } template std::complex SumP2(unsigned polarization) const { return std::complex(sumP2[polarization].real(), sumP2[polarization].imag()); } template std::complex DMean(unsigned polarization) const { return std::complex(dSum[polarization].real() / dCount[polarization], dSum[polarization].imag() / dCount[polarization]); } template std::complex DSum(unsigned polarization) const { return std::complex(dSum[polarization].real(), dSum[polarization].imag()); } template std::complex DSumP2(unsigned polarization) const { return std::complex(dSumP2[polarization].real(), dSumP2[polarization].imag()); } unsigned long* rfiCount; unsigned long* count; std::complex* sum; std::complex* sumP2; unsigned long* dCount; std::complex* dSum; std::complex* dSumP2; private: void initialize() { rfiCount = new unsigned long[_polarizationCount]; count = new unsigned long[_polarizationCount]; sum = new std::complex[_polarizationCount]; sumP2 = new std::complex[_polarizationCount]; dCount = new unsigned long[_polarizationCount]; dSum = new std::complex[_polarizationCount]; dSumP2 = new std::complex[_polarizationCount]; } void destruct() { delete[] rfiCount; delete[] count; delete[] sum; delete[] sumP2; delete[] dCount; delete[] dSum; delete[] dSumP2; } unsigned _polarizationCount; }; #endif aoflagger-v3.5.1/quality/statisticscollection.cpp0000664000175000017500000003563314752462134020443 0ustar oleole#include "statisticscollection.h" template void StatisticsCollection::addTimeAndBaseline( unsigned antenna1, unsigned antenna2, double time, double centralFrequency, int polarization, const float* reals, const float* imags, const bool* isRFI, const bool* origFlags, unsigned nsamples, unsigned step, unsigned stepRFI, unsigned stepFlags) { unsigned long rfiCount = 0; unsigned long count = 0; long double sum_R = 0.0, sum_I = 0.0; long double sumP2_R = 0.0, sumP2_I = 0.0; for (unsigned j = 0; j < nsamples; ++j) { if (!*origFlags) { if (std::isfinite(*reals) && std::isfinite(*imags)) { if (*isRFI) { ++rfiCount; } else { const long double rVal = *reals; const long double iVal = *imags; ++count; sum_R += rVal; sum_I += iVal; sumP2_R += rVal * rVal; sumP2_I += iVal * iVal; } } } reals += step; imags += step; isRFI += stepRFI; origFlags += stepFlags; } if (antenna1 != antenna2) { DefaultStatistics& timeStat = getTimeStatistic(time, centralFrequency); addToStatistic(timeStat, polarization, count, sum_R, sum_I, sumP2_R, sumP2_I, rfiCount); } DefaultStatistics& baselineStat = getBaselineStatistic(antenna1, antenna2, centralFrequency); addToStatistic(baselineStat, polarization, count, sum_R, sum_I, sumP2_R, sumP2_I, rfiCount); } template void StatisticsCollection::addFrequency( unsigned band, int polarization, const float* reals, const float* imags, const bool* isRFI, const bool* origFlags, unsigned nsamples, unsigned step, unsigned stepRFI, unsigned stepFlags, bool shiftOneUp) { std::vector& bandStats = _bands.find(band)->second; const unsigned fAdd = shiftOneUp ? 1 : 0; for (unsigned j = 0; j < nsamples; ++j) { if (!*origFlags) { if (std::isfinite(*reals) && std::isfinite(*imags)) { DefaultStatistics& freqStat = *bandStats[j + fAdd]; if (*isRFI) { addToStatistic(freqStat, polarization, 0, 0.0, 0.0, 0.0, 0.0, 1); } else { const long double r = *reals, i = *imags; addToStatistic(freqStat, polarization, 1, r, i, r * r, i * i, 0); } } } isRFI += stepRFI; origFlags += stepFlags; reals += step; imags += step; } } void StatisticsCollection::Add(unsigned antenna1, unsigned antenna2, double time, unsigned band, int polarization, const float* reals, const float* imags, const bool* isRFI, const bool* origFlags, unsigned nsamples, unsigned step, unsigned stepRFI, unsigned stepFlags) { if (nsamples == 0) return; const double centralFrequency = _centralFrequencies.find(band)->second; addTimeAndBaseline(antenna1, antenna2, time, centralFrequency, polarization, reals, imags, isRFI, origFlags, nsamples, step, stepRFI, stepFlags); if (antenna1 != antenna2) addFrequency(band, polarization, reals, imags, isRFI, origFlags, nsamples, step, stepRFI, stepFlags, false); // Allocate vector with length nsamples, so there is // a diff element, even if nsamples=1. std::vector diffReals(nsamples); std::vector diffImags(nsamples); bool* diffRFIFlags = new bool[nsamples]; bool* diffOrigFlags = new bool[nsamples]; for (unsigned i = 0; i < nsamples - 1; ++i) { diffReals[i] = (reals[(i + 1) * step] - reals[i * step]) * M_SQRT1_2; diffImags[i] = (imags[(i + 1) * step] - imags[i * step]) * M_SQRT1_2; diffRFIFlags[i] = isRFI[i * stepRFI] | isRFI[(i + 1) * stepRFI]; diffOrigFlags[i] = origFlags[i * stepFlags] | origFlags[(i + 1) * stepFlags]; } addTimeAndBaseline(antenna1, antenna2, time, centralFrequency, polarization, &(diffReals[0]), &(diffImags[0]), diffRFIFlags, diffOrigFlags, nsamples - 1, 1, 1, 1); if (antenna1 != antenna2) { addFrequency(band, polarization, &(diffReals[0]), &(diffImags[0]), diffRFIFlags, diffOrigFlags, nsamples - 1, 1, 1, 1, false); addFrequency(band, polarization, &(diffReals[0]), &(diffImags[0]), diffRFIFlags, diffOrigFlags, nsamples - 1, 1, 1, 1, true); } delete[] diffRFIFlags; delete[] diffOrigFlags; } void StatisticsCollection::AddToTimeFrequency( unsigned antenna1, unsigned antenna2, double time, unsigned band, int polarization, const float* reals, const float* imags, const bool* isRFI, const bool* origFlags, unsigned nsamples, unsigned step, unsigned stepRFI, unsigned stepFlags) { if (nsamples == 0) return; if (antenna1 == antenna2) return; addToTimeFrequency(time, &_bandFrequencies[band][0], polarization, reals, imags, isRFI, origFlags, nsamples, step, stepRFI, stepFlags, false); // Allocate vector with length nsamples, so there is // a diff element, even if nsamples=1. std::vector diffReals(nsamples); std::vector diffImags(nsamples); bool* diffRFIFlags = new bool[nsamples]; bool* diffOrigFlags = new bool[nsamples]; for (unsigned i = 0; i < nsamples - 1; ++i) { diffReals[i] = (reals[(i + 1) * step] - reals[i * step]) * M_SQRT1_2; diffImags[i] = (imags[(i + 1) * step] - imags[i * step]) * M_SQRT1_2; diffRFIFlags[i] = isRFI[i * stepRFI] | isRFI[(i + 1) * stepRFI]; diffOrigFlags[i] = origFlags[i * stepFlags] | origFlags[(i + 1) * stepFlags]; } addToTimeFrequency(time, &_bandFrequencies[band][0], polarization, &(diffReals[0]), &(diffImags[0]), diffRFIFlags, diffOrigFlags, nsamples - 1, 1, 1, 1, false); addToTimeFrequency(time, &_bandFrequencies[band][0], polarization, &(diffReals[0]), &(diffImags[0]), diffRFIFlags, diffOrigFlags, nsamples - 1, 1, 1, 1, true); delete[] diffRFIFlags; delete[] diffOrigFlags; } void StatisticsCollection::AddImage(unsigned antenna1, unsigned antenna2, const double* times, unsigned band, int polarization, const Image2DCPtr& realImage, const Image2DCPtr& imagImage, const Mask2DCPtr& rfiMask, const Mask2DCPtr& correlatorMask) { if (realImage->Width() == 0 || realImage->Height() == 0) return; const double centralFrequency = _centralFrequencies.find(band)->second; DefaultStatistics& baselineStat = getBaselineStatistic(antenna1, antenna2, centralFrequency); std::vector& bandStats = _bands.find(band)->second; std::vector timeStats(realImage->Width()); for (size_t t = 0; t != realImage->Width(); ++t) timeStats[t] = &getTimeStatistic(times[t], centralFrequency); for (size_t f = 0; f < realImage->Height(); ++f) { DefaultStatistics& freqStat = *bandStats[f]; const bool *origFlags = correlatorMask->ValuePtr(0, f), *nextOrigFlags = origFlags + correlatorMask->Stride(), *isRFI = rfiMask->ValuePtr(0, f), *isNextRFI = isRFI + rfiMask->Stride(); const float *reals = realImage->ValuePtr(0, f), *imags = imagImage->ValuePtr(0, f), *nextReal = reals + realImage->Stride(), *nextImag = imags + imagImage->Stride(); for (size_t t = 0; t < realImage->Width(); ++t) { if (!*origFlags && std::isfinite(*reals) && std::isfinite(*imags)) { long double real = *reals, imag = *imags; if (*isRFI) { if (antenna1 != antenna2) { ++timeStats[t]->rfiCount[polarization]; ++freqStat.rfiCount[polarization]; } ++baselineStat.rfiCount[polarization]; } else { long double realSq = real * real, imagSq = imag * imag; if (antenna1 != antenna2) { addSingleNonRFISampleToStatistic(*timeStats[t], polarization, real, imag, realSq, imagSq); addSingleNonRFISampleToStatistic(freqStat, polarization, real, imag, realSq, imagSq); } addSingleNonRFISampleToStatistic(baselineStat, polarization, real, imag, realSq, imagSq); } if (f != realImage->Height() - 1) { DefaultStatistics& nextFreqStat = *bandStats[f + 1]; if (!*nextOrigFlags && std::isfinite(*nextReal) && std::isfinite(*nextImag)) { real = (*nextReal - *reals) * M_SQRT1_2; imag = (*nextImag - *imags) * M_SQRT1_2; if (!(*isRFI || *isNextRFI)) { long double realSq = real * real, imagSq = imag * imag; if (antenna1 != antenna2) { addSingleNonRFISampleToStatistic( *timeStats[t], polarization, real, imag, realSq, imagSq); addSingleNonRFISampleToStatistic( freqStat, polarization, real, imag, realSq, imagSq); addSingleNonRFISampleToStatistic( nextFreqStat, polarization, real, imag, realSq, imagSq); } addSingleNonRFISampleToStatistic( baselineStat, polarization, real, imag, realSq, imagSq); } } } } ++origFlags; ++isRFI; ++reals; ++imags; ++nextOrigFlags; ++isNextRFI; ++nextReal; ++nextImag; } } } void StatisticsCollection::lowerResolution( StatisticsCollection::DoubleStatMap& map, size_t maxSteps) const { if (map.size() > maxSteps) { DoubleStatMap newMap; double gridStep, gridStart; if (maxSteps > 1) { const double oldGridStep = (map.rbegin()->first - map.begin()->first) / (map.size() - 1); gridStep = (map.rbegin()->first - map.begin()->first + oldGridStep) / maxSteps; gridStart = map.begin()->first - 0.5 * oldGridStep; } else { gridStep = map.rbegin()->first - map.begin()->first; gridStart = map.begin()->first; } size_t gridIndex = 0; for (DoubleStatMap::iterator i = map.begin(); i != map.end();) { DefaultStatistics integratedStat(_polarizationCount); double cellMid = (gridIndex + 0.5) * gridStep + gridStart, cellEnd = (gridIndex + 1) * gridStep + gridStart; size_t count = 0; while (i != map.end() && i->first < cellEnd) { ++count; integratedStat += i->second; ++i; } ++gridIndex; // If the last items are not yet gridded, they might be just over the // border due to rounding errors; put them in the last bucket: if (gridIndex == maxSteps) { while (i != map.end()) { ++count; integratedStat += i->second; ++i; } } if (count > 0) newMap.insert( std::pair(cellMid, integratedStat)); } map = newMap; } } template void StatisticsCollection::addToTimeFrequency( double time, const double* frequencies, int polarization, const float* reals, const float* imags, const bool* isRFI, const bool* origFlags, unsigned nsamples, unsigned step, unsigned stepRFI, unsigned stepFlags, bool shiftOneUp) { const unsigned fAdd = shiftOneUp ? 1 : 0; for (unsigned j = 0; j < nsamples; ++j) { if (!*origFlags) { if (std::isfinite(*reals) && std::isfinite(*imags)) { DefaultStatistics& timeStat = getTimeStatistic(time, frequencies[j + fAdd]); if (*isRFI) { addToStatistic(timeStat, polarization, 0, 0.0, 0.0, 0.0, 0.0, 1); } else { const long double r = *reals, i = *imags; addToStatistic(timeStat, polarization, 1, r, i, r * r, i * i, 0); } } } isRFI += stepRFI; origFlags += stepFlags; reals += step; imags += step; } } void StatisticsCollection::saveTime(QualityTablesFormatter& qd) const { initializeEmptyStatistics(qd, QualityTablesFormatter::TimeDimension); Indices indices; indices.fill(qd); StatisticSaver saver; saver.dimension = QualityTablesFormatter::TimeDimension; saver.qualityData = &qd; for (std::map::const_iterator j = _timeStatistics.begin(); j != _timeStatistics.end(); ++j) { saver.frequency = j->first; const DoubleStatMap& map = j->second; for (DoubleStatMap::const_iterator i = map.begin(); i != map.end(); ++i) { saver.time = i->first; const DefaultStatistics& stat = i->second; saveEachStatistic(saver, stat, indices); } } } void StatisticsCollection::saveFrequency(QualityTablesFormatter& qd) const { if (!_frequencyStatistics.empty()) { initializeEmptyStatistics(qd, QualityTablesFormatter::FrequencyDimension); Indices indices; indices.fill(qd); StatisticSaver saver; saver.dimension = QualityTablesFormatter::FrequencyDimension; saver.qualityData = &qd; for (DoubleStatMap::const_iterator i = _frequencyStatistics.begin(); i != _frequencyStatistics.end(); ++i) { saver.frequency = i->first; const DefaultStatistics& stat = i->second; saveEachStatistic(saver, stat, indices); } } } void StatisticsCollection::saveBaseline(QualityTablesFormatter& qd) const { if (!_baselineStatistics.empty()) { initializeEmptyStatistics(qd, QualityTablesFormatter::BaselineDimension); Indices indices; indices.fill(qd); StatisticSaver saver; saver.dimension = QualityTablesFormatter::BaselineDimension; saver.frequency = centralFrequency(); saver.qualityData = &qd; for (std::map::const_iterator j = _baselineStatistics.begin(); j != _baselineStatistics.end(); ++j) { saver.frequency = j->first; const BaselineStatisticsMap& map = j->second; const std::vector> baselines = map.BaselineList(); for (std::vector>::const_iterator i = baselines.begin(); i != baselines.end(); ++i) { saver.antenna1 = i->first; saver.antenna2 = i->second; const DefaultStatistics& stat = map.GetStatistics(saver.antenna1, saver.antenna2); saveEachStatistic(saver, stat, indices); } } } } aoflagger-v3.5.1/ref/0000775000175000017500000000000015146315735012545 5ustar oleoleaoflagger-v3.5.1/ref/reffileexception.h0000664000175000017500000000056314752462134016253 0ustar oleole#ifndef AO_REFFILEEXCEPTION_H #define AO_REFFILEEXCEPTION_H #include namespace AOTools { class RefFileException : public std::runtime_error { public: RefFileException() : std::runtime_error("Exception in reference file") {} explicit RefFileException(const std::string& message) : std::runtime_error(message) {} }; } // namespace AOTools #endif aoflagger-v3.5.1/ref/reffileentry.h0000664000175000017500000000542614752462134015421 0ustar oleole#ifndef AO_REFFILEENTRY_H #define AO_REFFILEENTRY_H #include #include #include #include "reffileexception.h" namespace AOTools { class RefFileEntry { public: friend class RefFile; RefFileEntry() : _size(0) {} RefFileEntry(const RefFileEntry& source) : _path(source._path), _frequency(source._frequency), _size(source._size), _node(source._node) {} RefFileEntry& operator=(const RefFileEntry& source) { _path = source._path; _frequency = source._frequency; _size = source._size; _node = source._node; return *this; } const std::string& Path() const { return _path; } const std::string& Frequency() const { return _frequency; } unsigned Size() const { return _size; } const std::string& Node() const { return _node; } void SetPath(const std::string& path) { _path = path; } private: std::string _path; std::string _frequency; unsigned _size; std::string _node; bool read(std::istream& stream) { std::string line; do { if (!stream.good()) return false; std::getline(stream, line); if (stream.fail()) return false; if (stream.bad()) throw RefFileException("Error in IO"); } while (ignoreLine(line)); assignFromString(line); return true; } void write(std::ostream& stream) const { stream << _path << ' ' << _frequency << ' ' << _size << ' ' << _node << "\n"; } void assignFromString(const std::string& line) { std::string::const_iterator i = line.begin(); if (!getNextToken(_path, i, line.end())) throw RefFileException("Expecting a path"); if (!getNextToken(_frequency, i, line.end())) throw RefFileException("Expecting frequency description"); std::string sizeString; if (!getNextToken(sizeString, i, line.end())) throw RefFileException("Expecting a size"); _size = atoi(sizeString.c_str()); if (!getNextToken(_node, i, line.end())) throw RefFileException("Expecting a node"); } static bool ignoreLine(const std::string& line) { for (std::string::const_iterator i = line.begin(); i != line.end(); ++i) { if (*i == '#') return true; if (!ignorable(*i)) return false; } return true; } static bool ignorable(std::string::value_type ch) { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r'; } static bool getNextToken(std::string& dest, std::string::const_iterator& ptr, const std::string::const_iterator end) { std::ostringstream token; while (ptr != end && ignorable(*ptr)) ++ptr; if (ptr == end) return false; while (ptr != end && !ignorable(*ptr)) { token << *ptr; ++ptr; } dest = token.str(); return dest.size() != 0; } }; } // namespace AOTools #endif // AO_REFFILEENTRY_H aoflagger-v3.5.1/ref/reffile.h0000664000175000017500000000244514752462134014335 0ustar oleole#ifndef AO_REFFILE_H #define AO_REFFILE_H #include #include #include #include "reffileentry.h" #include "reffileexception.h" namespace AOTools { class RefFile { public: typedef std::vector::const_iterator const_iterator; RefFile() {} explicit RefFile(const std::string& refFilePath) { Read(refFilePath); } void Read(const std::string& refFilePath) { _entries.clear(); _refFilePath = refFilePath; std::ifstream file(_refFilePath.c_str()); RefFileEntry entry; while (entry.read(file)) { _entries.push_back(entry); } } void Write(std::ostream& destination) const { for (const_iterator i = begin(); i != end(); ++i) i->write(destination); } size_t Count() const { return _entries.size(); } const RefFileEntry& operator[](const size_t index) const { return _entries[index]; } void Add(const RefFileEntry& entry) { _entries.push_back(entry); } const_iterator begin() const { return _entries.begin(); } const_iterator end() const { return _entries.end(); } private: RefFile(const RefFile&) // don't allow copy {} void operator=(const RefFile&) // don't allow assignment {} std::vector _entries; std::string _refFilePath; }; } // namespace AOTools #endif // AO_REFFILE_H aoflagger-v3.5.1/interface/0000775000175000017500000000000015146315735013731 5ustar oleoleaoflagger-v3.5.1/interface/structures.h0000664000175000017500000000351414752462134016326 0ustar oleole#ifndef AOFLAGGER_STRUCTURES_H #define AOFLAGGER_STRUCTURES_H #include "../structures/image2d.h" #include "../structures/mask2d.h" #include "../structures/timefrequencymetadata.h" #include "../quality/histogramcollection.h" #include "../quality/statisticscollection.h" #include #include namespace aoflagger { class FlagMaskData { public: explicit FlagMaskData(Mask2DPtr theMask) : mask(theMask) {} Mask2DPtr mask; }; class ImageSetData { public: explicit ImageSetData(size_t initialSize) : images(initialSize), hasAntennas(false), hasInterval(false), hasBand(false), antenna1(0), antenna2(0), interval(0), band(0) {} std::vector images; bool hasAntennas, hasInterval, hasBand; size_t antenna1, antenna2, interval, band; }; class QualityStatisticsDataImp { public: QualityStatisticsDataImp(const double* _scanTimes, size_t nScans, size_t nPolarizations, bool _computeHistograms) : scanTimes(_scanTimes, _scanTimes + nScans), statistics(nPolarizations), histograms(nPolarizations), computeHistograms(_computeHistograms) {} std::vector scanTimes; StatisticsCollection statistics; HistogramCollection histograms; bool computeHistograms; }; class QualityStatisticsData { public: QualityStatisticsData(const double* _scanTimes, size_t nScans, size_t nPolarizations, bool computeHistograms) : _implementation(new QualityStatisticsDataImp( _scanTimes, nScans, nPolarizations, computeHistograms)) {} explicit QualityStatisticsData( std::shared_ptr implementation) : _implementation(implementation) {} std::shared_ptr _implementation; }; } // namespace aoflagger #endif aoflagger-v3.5.1/interface/flagmask.cpp0000664000175000017500000000275714752462134016233 0ustar oleole#include "aoflagger.h" #include "structures.h" namespace aoflagger { FlagMask::FlagMask() : _data(nullptr) {} FlagMask::FlagMask(size_t width, size_t height) : _data(new FlagMaskData(Mask2D::CreateUnsetMaskPtr(width, height))) {} FlagMask::FlagMask(size_t width, size_t height, bool initialValue) : _data(new FlagMaskData(Mask2D::CreateUnsetMaskPtr(width, height))) { if (initialValue) _data->mask->SetAll(); else _data->mask->SetAll(); } FlagMask::FlagMask(const FlagMask& sourceMask) : _data(sourceMask._data == nullptr ? nullptr : new FlagMaskData(*sourceMask._data)) { } FlagMask::FlagMask(FlagMask&& sourceMask) : _data(std::move(sourceMask._data)) {} FlagMask& FlagMask::operator=(const FlagMask& flagMask) { if (flagMask._data == nullptr) { _data.reset(); } else if (_data == nullptr) { _data.reset(new FlagMaskData(*flagMask._data)); } else { *_data = *flagMask._data; } return *this; } FlagMask& FlagMask::operator=(FlagMask&& flagMask) { std::swap(_data, flagMask._data); return *this; } FlagMask::~FlagMask() {} size_t FlagMask::Width() const { return _data->mask->Width(); } size_t FlagMask::Height() const { return _data->mask->Height(); } size_t FlagMask::HorizontalStride() const { return _data->mask->Stride(); } bool* FlagMask::Buffer() { return _data->mask->ValuePtr(0, 0); } const bool* FlagMask::Buffer() const { return _data->mask->ValuePtr(0, 0); } } // namespace aoflagger aoflagger-v3.5.1/interface/qualitystatistics.cpp0000664000175000017500000001031014752462134020231 0ustar oleole#ifndef QUALITY_STATISTICS_DATA_H #define QUALITY_STATISTICS_DATA_H #include "aoflagger.h" #include "structures.h" namespace aoflagger { QualityStatistics::QualityStatistics() : _data(nullptr) {} QualityStatistics::QualityStatistics(const double* scanTimes, size_t nScans, const double* channelFrequencies, size_t nChannels, size_t nPolarizations, bool computeHistograms) : _data(new QualityStatisticsData(scanTimes, nScans, nPolarizations, computeHistograms)) { _data->_implementation->statistics.InitializeBand(0, channelFrequencies, nChannels); } QualityStatistics::QualityStatistics(const QualityStatistics& sourceQS) : _data(sourceQS._data == nullptr ? nullptr : new QualityStatisticsData(sourceQS._data->_implementation)) {} QualityStatistics::QualityStatistics(QualityStatistics&& sourceQS) : _data(std::move(sourceQS._data)) { sourceQS._data = nullptr; } QualityStatistics::~QualityStatistics() {} QualityStatistics& QualityStatistics::operator=( const QualityStatistics& sourceQS) { if (_data == nullptr) { if (sourceQS._data != nullptr) _data.reset(new QualityStatisticsData(sourceQS._data->_implementation)); } else { if (sourceQS._data != nullptr) { _data->_implementation = sourceQS._data->_implementation; } else { _data.reset(); } } return *this; } QualityStatistics& QualityStatistics::operator=(QualityStatistics&& sourceQS) { if (_data == nullptr || sourceQS._data == nullptr) { std::swap(_data, sourceQS._data); } else { _data->_implementation = std::move(sourceQS._data->_implementation); } return *this; } QualityStatistics& QualityStatistics::operator+=(const QualityStatistics& rhs) { _data->_implementation->statistics.Add( rhs._data->_implementation->statistics); _data->_implementation->histograms.Add( rhs._data->_implementation->histograms); return *this; } void QualityStatistics::CollectStatistics(const ImageSet& imageSet, const FlagMask& rfiFlags, const FlagMask& correlatorFlags, size_t antenna1, size_t antenna2) { StatisticsCollection& stats(_data->_implementation->statistics); HistogramCollection& histograms(_data->_implementation->histograms); const std::vector& times(_data->_implementation->scanTimes); if (imageSet.ImageCount() == 1) { stats.AddImage(antenna1, antenna2, ×[0], 0, 0, imageSet._data->images[0], imageSet._data->images[0], rfiFlags._data->mask, correlatorFlags._data->mask); if (_data->_implementation->computeHistograms) { histograms.Add(antenna1, antenna2, 0, imageSet._data->images[0], rfiFlags._data->mask, correlatorFlags._data->mask); } } else { const size_t polarizationCount = imageSet.ImageCount() / 2; for (size_t polarization = 0; polarization != polarizationCount; ++polarization) { stats.AddImage(antenna1, antenna2, ×[0], 0, polarization, imageSet._data->images[polarization * 2], imageSet._data->images[polarization * 2 + 1], rfiFlags._data->mask, correlatorFlags._data->mask); if (_data->_implementation->computeHistograms) { histograms.Add(antenna1, antenna2, polarization, imageSet._data->images[polarization * 2], imageSet._data->images[polarization * 2 + 1], rfiFlags._data->mask, correlatorFlags._data->mask); } } } } void QualityStatistics::WriteStatistics( const std::string& measurementSetPath) const { QualityTablesFormatter qFormatter(measurementSetPath); _data->_implementation->statistics.Save(qFormatter); HistogramCollection& histograms(_data->_implementation->histograms); if (!histograms.Empty()) { HistogramTablesFormatter hFormatter(measurementSetPath); histograms.Save(hFormatter); } } } // end of namespace aoflagger #endif aoflagger-v3.5.1/interface/strategy.cpp0000664000175000017500000001474614752462134016311 0ustar oleole#include "aoflagger.h" #include "structures.h" #include "../lua/luastrategy.h" #include "../lua/scriptdata.h" #include "../util/progress/progresslistener.h" #include "../structures/timefrequencydata.h" #include namespace aoflagger { class ErrorListener final : public ProgressListener { void OnStartTask(const std::string&) override {} void OnProgress(size_t, size_t) override {} void OnFinish() override {} void OnException(std::exception& e) override { std::cerr << "*** EXCEPTION OCCURED IN THE AOFLAGGER ***\n" "The AOFlagger encountered a bug or the given strategy was " "invalid!\n" "The reported exception " << typeid(e).name() << " is:\n" << e.what(); } }; class ForwardingListener final : public ProgressListener { public: explicit ForwardingListener(StatusListener* destination) : _destination(destination) {} void OnStartTask(const std::string& description) override { _destination->OnStartTask(description); } void OnProgress(size_t progress, size_t maxProgress) override { _destination->OnProgress(progress, maxProgress); } void OnFinish() override { _destination->OnFinish(); } void OnException(std::exception& thrownException) override { _destination->OnException(thrownException); } private: StatusListener* _destination; }; class StrategyData { public: [[no_unique_address]] LuaStrategy _lua; }; Strategy::Strategy() : _data(), _aoflagger(nullptr) {} Strategy::Strategy(const std::string& filename, AOFlagger* aoflagger) : _data(new StrategyData()), _aoflagger(aoflagger) { _data->_lua.Initialize(); _data->_lua.LoadFile(filename.c_str()); } Strategy::Strategy(Strategy&& sourceStrategy) : _data(sourceStrategy._data ? new StrategyData(std::move(*sourceStrategy._data)) : nullptr), _aoflagger(sourceStrategy._aoflagger) {} Strategy::~Strategy() {} Strategy Strategy::makeFromString(const std::string& script, AOFlagger* aoflagger) { Strategy strategy; strategy._data.reset(new StrategyData()); strategy._aoflagger = aoflagger; strategy._data->_lua.Initialize(); strategy._data->_lua.LoadText(script); return strategy; } Strategy& Strategy::operator=(Strategy&& sourceStrategy) { if (sourceStrategy._data) _data = std::move(sourceStrategy._data); else _data = nullptr; _aoflagger = sourceStrategy._aoflagger; return *this; } inline static AntennaInfo ConvertAntenna(Antenna& antenna) { AntennaInfo antennaInfo; antennaInfo.diameter = antenna.diameter; antennaInfo.id = antenna.id; antennaInfo.mount = antenna.mount; antennaInfo.name = antenna.name; antennaInfo.station = antenna.station; antennaInfo.position.x = antenna.x; antennaInfo.position.y = antenna.y; antennaInfo.position.z = antenna.z; return antennaInfo; } inline static BandInfo ConvertBand(Band& band) { BandInfo bandInfo; bandInfo.windowIndex = band.id; bandInfo.channels.resize(band.channels.size()); for (size_t i = 0; i != band.channels.size(); ++i) { bandInfo.channels[i].frequencyIndex = i; bandInfo.channels[i].frequencyHz = band.channels[i].frequency; bandInfo.channels[i].channelWidthHz = band.channels[i].width; bandInfo.channels[i].effectiveBandWidthHz = band.channels[i].width; bandInfo.channels[i].resolutionHz = band.channels[i].width; } return bandInfo; } FlagMask Strategy::Run(const ImageSet& input, const FlagMask& preExistingFlags) { return run(input, &preExistingFlags); } FlagMask Strategy::Run(const ImageSet& input) { return run(input, nullptr); } FlagMask Strategy::run(const ImageSet& input, const FlagMask* preExistingFlags) { std::unique_ptr listener; if (_aoflagger->_statusListener == nullptr) listener.reset(new ErrorListener()); else listener.reset(new ForwardingListener(_aoflagger->_statusListener)); Mask2DCPtr inputMask; if (preExistingFlags == nullptr) inputMask = Mask2D::CreateSetMaskPtr(input.Width(), input.Height()); else inputMask = preExistingFlags->_data->mask; TimeFrequencyData inputData; const Image2DPtr zeroImage = Image2D::CreateZeroImagePtr(input.Width(), input.Height()); switch (input.ImageCount()) { case 1: inputData = TimeFrequencyData(TimeFrequencyData::AmplitudePart, aocommon::Polarization::StokesI, input._data->images[0]); inputData.SetGlobalMask(inputMask); break; case 2: inputData = TimeFrequencyData(aocommon::Polarization::StokesI, input._data->images[0], input._data->images[1]); inputData.SetGlobalMask(inputMask); break; case 4: inputData = TimeFrequencyData(aocommon::Polarization::XX, input._data->images[0], input._data->images[1], aocommon::Polarization::YY, input._data->images[2], input._data->images[3]); inputData.SetIndividualPolarizationMasks(inputMask, inputMask); break; case 8: inputData = TimeFrequencyData::FromLinear( input._data->images[0], input._data->images[1], input._data->images[2], input._data->images[3], input._data->images[4], input._data->images[5], input._data->images[6], input._data->images[7]); inputData.SetIndividualPolarizationMasks(inputMask, inputMask, inputMask, inputMask); break; } const TimeFrequencyMetaDataPtr metaData(new TimeFrequencyMetaData()); if (input.HasAntennas() && !_aoflagger->_antennas.empty()) { metaData->SetAntenna1( ConvertAntenna(_aoflagger->_antennas[input.Antenna1()])); metaData->SetAntenna2( ConvertAntenna(_aoflagger->_antennas[input.Antenna2()])); } if (input.HasBand() && !_aoflagger->_bands.empty()) { metaData->SetBand(ConvertBand(_aoflagger->_bands[input.Band()])); } if (input.HasInterval() && !_aoflagger->_intervals.empty()) { metaData->SetObservationTimes( _aoflagger->_intervals[input.Interval()].times); } ScriptData scriptData; scriptData.SetProgressListener(*listener); _data->_lua.Execute(inputData, metaData, scriptData, "execute"); listener.reset(); inputMask.reset(); FlagMask flagMask; flagMask._data.reset( new FlagMaskData(Mask2DPtr(new Mask2D(*inputData.GetSingleMask())))); return flagMask; } } // namespace aoflagger aoflagger-v3.5.1/interface/aoflagger.h0000664000175000017500000010075314752462134016035 0ustar oleole/** @file aoflagger.h @brief Main AOFlagger header file. * @author André Offringa offringa@gmail.com * @copyright by A.R. Offringa under the GPL version 3 */ #ifndef AOFLAGGER_INTERFACE_H #define AOFLAGGER_INTERFACE_H #include #include #include #include #include /** @brief Contains all the public types used by the AOFlagger. * * See the @ref AOFlagger class description for details. * @author André Offringa offringa@gmail.com */ namespace aoflagger { /** @brief Strategy identifier for the supported telescopes. * * If you have an optimized strategy for an unlisted telescope, please * contact me. * @sa AOFlagger::FindStrategyFile(). * @since Version 3.0 */ enum class TelescopeId { /** @brief Most generic strategy. */ GENERIC_TELESCOPE, /** @brief The AARTFAAC telescope, correlating the superterp antennas of LOFAR. */ AARTFAAC_TELESCOPE, /** @brief The WSRT telescope with the Apertif focal plane array receiver system. */ APERTIF_TELESCOPE, /** @brief Arecibo radio telescope, the 305 m telescope in Puerto Rico. */ ARECIBO_TELESCOPE, /** @brief Australian Telescope Compact Array in Australia. */ ATCA_TELESCOPE, /** @brief Bighorns, instrument aimed at achieving an averaged all-sky measurement of the Epoch of Reionisation signal. */ BIGHORNS_TELESCOPE, /** @brief JVLA, the Jansky Very Large Array in New Mexico. */ JVLA_TELESCOPE, /** @brief LOFAR. the Low-Frequency Array in Europe. */ LOFAR_TELESCOPE, /** @brief MWA, the Murchison Widefield Array in Western Australia. */ MWA_TELESCOPE, /** @brief NenuFAR, the New Extension in Nançay upgrading LOFAR. */ NENUFAR_TELESCOPE, /** @brief Parkes, the single dish telescope in New South Wales. */ PARKES_TELESCOPE, /** @brief WSRT, the Westerbork Synthesis Radio Telescope in the Netherlands. */ WSRT_TELESCOPE }; /** * @brief Description of a single antenna. */ struct Antenna { /** @brief A unique identifying number for this antenna. */ size_t id; /** @brief ITRF position of the antenna. May be zero if unknown. */ double x, y, z; /** @brief Name of the antenna, e.g. "RT5". May be empty if unknown. */ std::string name; /** @brief Diameter of the antenna in meters. May be zero if unknown. */ double diameter; /** @brief Type of mount, e.g. "FIXED". May be empty if unknown. */ std::string mount; /** @brief Station to which this antenna belongs. May be empty if unknown. */ std::string station; }; /** * @brief A single channel. */ struct Channel { /** @brief Central frequency of this channel. */ double frequency; /** @brief Width of this channel. */ double width; }; /** * @brief Description of a single band. */ struct Band { /** @brief A unique identifying number for this band. */ size_t id; /** @brief List of channels in this band. */ std::vector channels; }; /** @brief A list of time value for a consecutive interval. */ struct Interval { size_t id; /** @brief Time values (MJD). */ std::vector times; }; /** @brief A set of time-frequency 'images' which together contain data for one * correlated baseline or dish. * * The class either holds 1, 2, 4 or 8 images. These images have time on the * x-axis (most rapidly changing index) and frequency on the y-axis. The * cells specify flux levels, which do not need to have been calibrated. * * If the set contains only one image, it specifies amplitudes of a single * polarization. If it contains two images, it specifies the real and imaginary * parts of a single polarization. With four images, it contains the real * and imaginary values of two polarizations (ordered real pol A, imag pol A, * real pol B, imag pol B). With eight images, it contains complex values for * four correlated polarizations (ordered real pol A, imag pol A, real pol B, * ... etc). * * @note When accesses the image data, note that there might be more items on * one row than the width of the image. The rows are padded to align them e.g. * for SSE instructions. Use @ref HorizontalStride() to get the actual number of * floats per row. */ class ImageSet { public: friend class AOFlagger; friend class QualityStatistics; friend class Strategy; /** @brief Construct an empty ImageSet. * * The only operations allowed on an empty ImageSet are to assign to it. Use * AOFlagger::MakeImageSet() to construct a non-empty ImageSet. */ ImageSet(); /** @brief Copy the image set. Only references to images are copied. */ ImageSet(const ImageSet& sourceImageSet); /** @brief Move from the image set. * @since Version 2.10 */ ImageSet(ImageSet&& sourceImageSet); /** @brief Destruct image set. Destroys its images if no longer referenced. */ ~ImageSet(); /** @brief Assign to this image set. Only references to images are copied. */ ImageSet& operator=(const ImageSet& sourceImageSet); /** @brief Move assign to this image set. * @since Version 2.10 */ ImageSet& operator=(ImageSet&& sourceImageSet); /** @brief Get access to the data buffer of an image. * @param imageIndex Index of image. See class description for ordering. * \note Rows are padded, see @ref HorizontalStride(). */ float* ImageBuffer(size_t imageIndex); /** @brief Get constant access to the data buffer of an image. * @param imageIndex Index of image. See class description for ordering. * \note Rows are padded, see @ref HorizontalStride(). */ const float* ImageBuffer(size_t imageIndex) const; /** @brief Get width (number of time steps) of images. */ size_t Width() const; /** @brief Get height (number of frequency channels) of images. */ size_t Height() const; /** @brief Get number of images, see class description for details. */ size_t ImageCount() const; /** @brief Get total number of floats in one row. * * Row might have been padded to allow for * SSE instructions and other optimizations. Therefore, one should * add the horizontal stride to a data pointer to get the float in the next * row (channel). * * Example: * @code{.cpp} *(ImageSet::ImageBuffer(imageIndex) + x + y * ImageSet::HorizontalStride()) * @endcode * will return the value at position x,y. */ size_t HorizontalStride() const; /** @brief Set all samples to the specified value. * @param newValue The new value for all values of all images in the set. * @since 2.5 */ void Set(float newValue); /** @brief Resize the image without reallocating new memory. * * This function allows to quickly change the dimension of the images in the * imageset. The new width has to fit in the image capacity as specified * during creation. When flagging many images of "almost" the same size, using * this method to change the size of images is drastically faster compared * to freeing and then allocating new images. It was added after rather * severe memory fragmentation problems in the Cotter MWA pipeline. * @param newWidth The new width of the images. Should satisfy newWidth <= * HorizontalStride(). * @since 2.5 */ void ResizeWithoutReallocation(size_t newWidth) const; void SetAntennas(size_t antenna1Index, size_t antenna2Index); bool HasAntennas() const; size_t Antenna1() const; size_t Antenna2() const; void SetInterval(size_t index); bool HasInterval() const; size_t Interval() const; void SetBand(size_t index); bool HasBand() const; size_t Band() const; private: ImageSet(size_t width, size_t height, size_t count); ImageSet(size_t width, size_t height, size_t count, float initialValue); ImageSet(size_t width, size_t height, size_t count, size_t widthCapacity); ImageSet(size_t width, size_t height, size_t count, float initialValue, size_t widthCapacity); static void assertValidCount(size_t count); std::unique_ptr _data; }; /** @brief A two-dimensional flag mask. * * The flag mask specifies which values in an @ref ImageSet are flagged. * A value @c true means a value is flagged, i.e., contains RFI and should * not be used in further data processing (calibration, imaging, etc.). * A flag denotes that the value at that time-frequency position should * be ignored for all polarizations. This normally makes sense, because if one * polarization is contaminated by RFI, all polarizations are probably * affected. Also, solving for Stokes matrices during calibration might * not work well when the polarizations are not flagged equally. * * If polarization-specific flags are needed, one could run the flagger on * each polarization individually. However, note that some algorithms, like * the morphological scale-invariant rank operator (SIR operator), work best * when seeing the flags from all polarizations. * * @note When accesses the flag data, note that there might be more items on one * row than the width of the mask. The rows are padded to align them e.g. for * SSE instructions. Use @ref HorizontalStride() to get the actual number of * bools per row. */ class FlagMask { public: friend class AOFlagger; friend class QualityStatistics; friend class Strategy; /** @brief Construct an empty FlagMask. * The properties of an empty FlagMask can not be accessed. */ FlagMask(); /** @brief Copy a flag mask. Only copies a reference, not the data. */ FlagMask(const FlagMask& sourceMask); /** @brief Move construct a flag mask. * @since Version 2.10 */ FlagMask(FlagMask&& sourceMask); /** @brief Copy assignment. * @since Version 2.10 */ FlagMask& operator=(const FlagMask& source); /** @brief Move assignment. * @since Version 2.10 */ FlagMask& operator=(FlagMask&& source); /** @brief Destroy a flag mask. Destroys mask data if no longer references. */ ~FlagMask(); /** @brief Get the width of the mask. */ size_t Width() const; /** @brief Get the height of the mask. */ size_t Height() const; /** @brief Get total number of bools in one row. * * Row might have been padded to allow for * SSE instructions and other optimizations. Therefore, one should * add the horizontal stride to a data pointer to get the flags in * the next row (channel). * * Example: * @code{.cpp} *(FlagMask::Buffer() + x + y * Buffer::HorizontalStride()) * @endcode * will return the flag value at position x,y. */ size_t HorizontalStride() const; /** @brief Get access to the data buffer. * @note The buffer is padded, see @ref HorizontalStride(). */ bool* Buffer(); /** @brief Get constant access to the data buffer. * @note The buffer is padded, see @ref HorizontalStride(). */ const bool* Buffer() const; private: FlagMask(size_t width, size_t height); FlagMask(size_t width, size_t height, bool initialValue); std::unique_ptr _data; }; /** @brief Holds a flagging strategy. * * Default "stock" strategies can be found with * @ref AOFlagger::FindStrategyFile(), and these or custom Lua files can * be loaded from disc with @ref AOFlagger::LoadStrategyFile(). * A user can create strategies with the @c rfigui tool that is part * of the aoflagger package. * * When flagging a large number of baselines it is recommended to use multiple * threads. This class is itself not thread save, but it is safe to use * different Strategy objects from different thread contexts. */ class Strategy { public: friend class AOFlagger; /** @brief Construct an empty strategy. * * The only operations allowed on an empty Strategy are to assign to it. Use * e.g. AOFlagger::LoadStrategyFile() to construct a non-empty Strategy. * @since Version 3 */ Strategy(); /** @brief Move construct a strategy. * @since Version 2.10 */ Strategy(Strategy&& sourceStrategy); /** @brief Destruct strategy. */ ~Strategy(); /** @brief Move assign to strategy. * @since Version 2.10 */ Strategy& operator=(Strategy&& sourceStrategy); /** @brief Run the flagging strategy on the given data. * * The Lua strategy is executed single-threaded. This function is not * thread safe: To flag multiple imagesets simultaneously from different * threads, it is necessary to create a Strategy object for each thread. * * @param input The data to run the flagger on. * @return The flags identifying bad (RFI contaminated) data. * @since 3.0 */ FlagMask Run(const ImageSet& input); /** @brief Run the flagging strategy on the given data with existing flags. * * This method is similar to @ref Run(const ImageSet&), except * that it will pass existing flags (e.g. as set by the correlator) * to the flagging strategy, which in the case of bad data can do a better * job of finding RFI in the good data. * @p input parameter. The @p strategy parameter can be the * same for different threads. * @param input The data to run the flagger on. * @param existingFlags Flags that indicate what data are bad. * @return A flag mask that identifies bad (RFI contaminated) data. * @since 3.0 */ FlagMask Run(const ImageSet& input, const FlagMask& existingFlags); private: Strategy(const std::string& filename, class AOFlagger* aoflagger); Strategy(const Strategy& sourceStrategy) = delete; Strategy& operator=(const Strategy& sourceStrategy) = delete; static Strategy makeFromString(const std::string& script, class AOFlagger* aoflagger); FlagMask run(const ImageSet& input, const FlagMask* existingFlags); std::unique_ptr _data; class AOFlagger* _aoflagger; }; /** @brief Statistics that can be collected online and saved to a measurement * set. * * It is useful to collect some statistics during flagging, because all data * goes through memory at highest resolution. This class contains the collected * statistics and some meta data required for collecting. It can be created with * @ref AOFlagger::MakeQualityStatistics(). Statistics can be added to it with * @ref CollectStatistics(), and saved to disk with * @ref WriteStatistics(). * * This class does not allow viewing or modifying statistics, it only contains * the most basic form to collect statistics during flagging and writing them in * the (well-defined) quality statistic tables format. These statistics can be * viewed interactively with the @c aoqplot tool. * * Collecting statistics is not as expensive as flagging, but still takes some * time, so it is recommended to use multiple threads for collecting as well. * This class is however not thread save, but it is okay to use different * QualityStatistics objects from different thread contexts. During * finalization, the different objects can be combined with the operator+=() * method, and then in full written to the measurement set. */ class QualityStatistics { public: friend class AOFlagger; /** Construct a QualityStatistics with null state. * * An object created by this constructor can only be assigned to. * @since Version 2.13 */ QualityStatistics(); /** @brief Copy the object. This is fast; only references are copied. */ QualityStatistics(const QualityStatistics& sourceQS); /** @brief Move construct the object. * @since Version 2.10 */ QualityStatistics(QualityStatistics&& sourceQS); /** @brief Destruct the object. Data is destroyed if no more references exist. */ ~QualityStatistics(); /** @brief Assign to this object. This is fast; only references are copied. */ QualityStatistics& operator=(const QualityStatistics& sourceQS); /** @brief Move-assign this object. This is fast; only references are moved. * @since Version 2.10 */ QualityStatistics& operator=(QualityStatistics&& sourceQS); /** @brief Combine the statistics from the given object with the statistics in * this object. * * This is a relative expensive operation, so should only be used scarsely. It * can be used to combine the results of different threads, as explained in * the class description. * * It is safe to combine quality statistics with different meta data (scan * time count, channel count, etc.). When using this object again during * collecting (see @ref CollectStatistics()), after combining it with another * object, it will still use the meta data it was initialized with. */ QualityStatistics& operator+=(const QualityStatistics& rhs); /** @brief Collect statistics from time-frequency images and masks. * * This will update the statistics in this object so that it * represents the combination of previous collected data and the newly * given data. * * This function can be called from different thread context, as long as each * thread uses its own QualityStatistics object. * See the @ref QualityStatistics class documentation for further * multithreading info. * @param imageSet Data to collect statistics from * @param rfiFlags Flags set by the automatic RFI detector * @param correlatorFlags Flags that were set prior to RFI detector, e.g. * because of a broken antenna or correlator hickup. * @param antenna1 Index of the first antenna involved in this baseline. * @param antenna2 Index of the second antenna involved in this baseline. * @since 3.0 */ void CollectStatistics(const ImageSet& imageSet, const FlagMask& rfiFlags, const FlagMask& correlatorFlags, size_t antenna1, size_t antenna2); /** @brief Write collected statistics in standard tables to a measurement set. * @param measurementSetPath Path to measurement set to which the statistics * will be written. * @since 3.0 */ void WriteStatistics(const std::string& measurementSetPath) const; private: QualityStatistics(const double* scanTimes, size_t nScans, const double* channelFrequencies, size_t nChannels, size_t nPolarizations, bool computeHistograms); std::unique_ptr _data; }; /** * @brief A base class which callers can inherit from to be able to receive * progress updates and error messages. * * A status listener should be thread safe when the Run() method is called in * parallel with the same StatusListener object. */ class StatusListener { public: /** * @brief Virtual destructor. */ virtual ~StatusListener() {} /** * @brief This virtual method is called when a new task is started. * * Typically, a client could display a message saying that the given task * 'description' is started. * @param description Description of the task, e.g. "SumThreshold". * @since 3.0 */ virtual void OnStartTask(const std::string& description) {} /** * @brief Called to update current progress. * * This can be used to display a progress bar if the strategy would take a lot * of time. * @param progress Current progress * @param maxProgress Progress that is required to finish the current task. */ virtual void OnProgress(size_t progress, size_t maxProgress) {} /** * @brief Called when detection has completely finished. * * This function will always be called exactly once on success. It is not * called when an exception occurred. @ref OnException() is called in that * case. */ virtual void OnFinish() {} /** * @brief Called when an exception occurs during execution of the strategy. * * This can occur when the Lua script throws an error. * @param thrownException The exception that was thrown. */ virtual void OnException(std::exception& thrownException) = 0; }; /** @brief Main class for access to the flagger functionality. * * Software using the flagger should first create an instance of the @ref * AOFlagger class, from which other actions can be initiated. * * ### Overview * * To flag a data set: * - Create the AOFlagger instance * - To use a stock strategy, call FindStrategyFile() * - Load and parse the strategy with LoadStrategyFile(), once for each thread * that will run. * - Create data buffers with MakeImageSet() * - For each correlated baseline or dish: * - - Fill the images with data from this correlated baseline or dish * - - Call Strategy::Run() with the created Strategy and ImageSet * - - Process the data that was returned in the FlagMask. * * Optionally, it is possible to assemble quality statistics that can be written * to the measurement set in the standard format that e.g. the @c aoqplot tool * can read. To do this: * - Create (once) a quality statistics object with MakeQualityStatistics(). * - After flagging a baseline, add it to the statistics object with * QualityStatistics::CollectStatistics(). * A "correlator mask" can be specified that describes which flags are not due * to RFI but caused by different things. * - When a full set is processed, store the statistics with WriteStatistics(). * * To flag multiple baselines, the Strategy and/or ImageSet objects can be * reused. * * ### Thread safety * * Each Strategy object runs in its own context, and will not perform any * unsynchronised writes to global variables. It is therefore safe to call * Strategy::Run() from different threads, as long as each thread uses its own * Strategy and ImageSet instances. QualityStatistics::CollectStatistics() is * also thread safe, as long as different QualityStatistics instances are * passed. For multi-threading, each thread should collect into its own * QualityStatistics object. When finished, these can be combined with * QualityStatistics::operator+=(). * * It is safe to create multiple AOFlagger instances, but not recommended. * * ### Data order * * A common problem for integrating the flagger, is that data are stored in a * different order: the time dimension * is often the direction with the slowest increasing indices. Because the * flagger needs one baseline at a time, this requires reordering the data. As * long as the data fits in memory, this reordering is quite straightforward. * When this is not the case, the data could be split into sub-bands and/or time * windows. Next, these parts can be passed to the flagger and recombined later * (if desired). * * To decide how to split, keep in mind that the flagger * works best when both a lot of channels and a lot of * timesteps are available. As an example: LOFAR splits into subbands of ~64 * channels, and the default processing with NDPPP loads as many timesteps as * possible in memory for flagging. Typically, this means at least a few hundred * of timesteps are processed at a time (with 1-3s per timestep), and this seems * to work well. * * The 'aoflagger' executable flags by default on the full measurement set. * For sets that are larger than memory, a mode is used in * which the data is reordered to disk before the actual flagging starts. It * turns out that this is much faster than reading each baseline directly from * the set, and it simultaneously produces the best flagging accuracy, so if * enough processing power is available to do so, this is another approach. */ class AOFlagger { public: /** @brief Create and initialize the flagger main class. */ AOFlagger() : _statusListener(nullptr) {} /** @brief Destructor. */ ~AOFlagger() {} /** @brief Create a new uninitialized @ref ImageSet with specified specs. * * The float values will not be initialized. * @param width Number of time steps in images * @param height Number of frequency channels in images * @param count Number of images in set (see class description * of @ref ImageSet for image order). * @return A new ImageSet. */ ImageSet MakeImageSet(size_t width, size_t height, size_t count) { return ImageSet(width, height, count); } /** @brief Create a new uninitialized @ref ImageSet with specified specs. * * The float values will not be initialized. * @param width Number of time steps in images * @param height Number of frequency channels in images * @param count Number of images in set (see class description * of @ref ImageSet for image order). * @param widthCapacity Allow for enlarging image to this size, @sa * ImageSet::ResizeWithoutReallocation() * @return A new ImageSet. * @since 2.6 */ ImageSet MakeImageSet(size_t width, size_t height, size_t count, size_t widthCapacity) { return ImageSet(width, height, count, widthCapacity); } /** @brief Create a new initialized @ref ImageSet with specified specs. * @param width Number of time steps in images * @param height Number of frequency channels in images * @param count Number of images in set (see class description * of @ref ImageSet for image order). * @param initialValue Initialize all pixels with this value. * @return A new ImageSet. */ ImageSet MakeImageSet(size_t width, size_t height, size_t count, float initialValue) { return ImageSet(width, height, count, initialValue); } /** @brief Create a new initialized @ref ImageSet with specified specs. * @param width Number of time steps in images * @param height Number of frequency channels in images * @param count Number of images in set (see class description * of @ref ImageSet for image order). * @param initialValue Initialize all pixels with this value. * @param widthCapacity Allow for enlarging image to this size, @sa * ImageSet::ResizeWithoutReallocation() * @return A new ImageSet. * @since 2.6 */ ImageSet MakeImageSet(size_t width, size_t height, size_t count, float initialValue, size_t widthCapacity) { return ImageSet(width, height, count, initialValue, widthCapacity); } /** @brief Create a new uninitialized @ref FlagMask with specified dimensions. * @param width Width of mask (number of timesteps) * @param height Height of mask (number of frequency channels) * @return A new FlagMask. */ FlagMask MakeFlagMask(size_t width, size_t height) { return FlagMask(width, height); } /** @brief Create a new initialized @ref FlagMask with specified dimensions. * @param width Width of mask (number of timesteps) * @param height Height of mask (number of frequency channels) * @param initialValue Value to initialize the mask to. * @return A new FlagMask. */ FlagMask MakeFlagMask(size_t width, size_t height, bool initialValue) { return FlagMask(width, height, initialValue); } /** @brief Find a Lua strategy for a specific telescope. * * The scenario name can be used to * distinguish different strategies for the same telescope, for example * to distinguish between different bands, different processing stages (e.g. * before or after averaging), different antennas (LOFAR HBA vs LBA), etc. * * This will search the data directory of the installation path of * aoflagger for a file named <Telescope_Name>-<scenario>.lua, or * <Telescope_Name>-default.lua in case scenario is empty. * * @param telescopeId Identifies the telescope to optimize the strategy for. * @param scenario A scenario name that should be searched for. * @returns Filename of strategy, or empty string if none found. * @since Version 3.0 */ std::string FindStrategyFile( enum TelescopeId telescopeId = TelescopeId::GENERIC_TELESCOPE, const std::string& scenario = ""); /** @brief Load a strategy from disk. * * The best way to create strategies is to use the @c rfigui tool. In case you * have optimized strategies for an unlisted telescope or for new scenarios, * please consider providing the file so I can add them to the repository. * * @param filename Full pathname to .lua strategy file. * @return The new @ref Strategy. * @since Version 3.0 */ Strategy LoadStrategyFile(const std::string& filename) { return Strategy(filename, this); } /** @brief Load a strategy from a string containing a Lua script. * * Similar to @ref LoadStrategyFile(). * * @param script String containing full Lua script. * @return The new @ref Strategy. * @since Version 3.1 */ Strategy LoadStrategyString(const std::string& script) { return Strategy::makeFromString(script, this); } /** * @brief Create a new object for collecting statistics. * @param scanTimes Array with times. The number of elements should match the * dimension of the time axis in calls to * QualityStatistics::CollectStatistics(). Each time is a MJD time in seconds. * @param nScans Number of elements in the @c scanTimes array. * @param channelFrequencies Frequency in Hz of each channel. The number of * elements should match the frequency axis in calls to * QualityStatistics::CollectStatistics(). * @param nChannels Number of elements in the @c channelFrequencies array. * @param nPolarizations Number of polarizations in the set (1, 2 or 4). * * See the QualityStatistics class description for info on multithreading * and/or combining statistics with different meta data. The meta data that is * passed to this method will be used for all calls to * QualityStatistics::CollectStatistics(). No histograms will be computed. */ QualityStatistics MakeQualityStatistics(const double* scanTimes, size_t nScans, const double* channelFrequencies, size_t nChannels, size_t nPolarizations); /** @brief Create a new object for collecting statistics, possibly with * histograms. * * See other overload of MakeQualityStatistics() for info. * @since Version 2.6 */ QualityStatistics MakeQualityStatistics( const double* scanTimes, size_t nScans, const double* channelFrequencies, size_t nChannels, size_t nPolarizations, bool computeHistograms); /** @brief Get the AOFlagger version number as a string. * @returns The version number, formatted like '1.2.3-subtitle', e.g. * '3.0-alpha'. * @since Version 2.6 */ static std::string GetVersionString(); /** @brief Get the AOFlagger version number separated in major, minor and * subminor fields. * @param major Most significant number of the version, e.g. '1' for version * '1.2.3'. This number is only incremented in major changes of the flagger. * @param minor Minor number of the version, e.g. '2' for version '1.2.3'. * This number is incremented for every public release. * @param subMinor Subminor number of the version, e.g. '3' for version * '1.2.3', or zero if the current version has no subminor number. This number * is incremented for internal releases or small bug fixes. * @since Version 2.6 */ static void GetVersion(short& major, short& minor, short& subMinor); /** @brief Get the date this version was released as a string. * @returns The version date formatted like "1982-05-08". * @since Version 2.6 */ static std::string GetVersionDate(); /** * @brief Set a handler for progress updates and exceptions. * * By default, exceptions will be reported to stderr and progress updates * will be ignored. If an application needs to handle either of these * themselves, they can override a @ref StatusListener that handles these * events and call this method to enable receiving the events. * This method is not thread safe. * @param statusListener The handler that will receive the status updates. * @since Version 2.7 */ void SetStatusListener(StatusListener* statusListener) { _statusListener = statusListener; } /** * Provide information about the antennas used in the data set. * This information is propagated to the Lua environment, * such that scripts can make choices based on the antenna. * It is not required to set this before flagging. */ void SetAntennaList(std::vector&& antennas) { _antennas = std::move(antennas); } const std::vector& AntennaList() const { return _antennas; } /** * Provide frequency information for the data set. * This information is propagated to the Lua environment, * such that scripts can make choices based on the antenna. * It is not required to set this before flagging. */ void SetBandList(std::vector&& bands) { _bands = std::move(bands); } const std::vector& BandList() const { return _bands; } void SetIntervalList(std::vector&& intervals) { _intervals = std::move(intervals); } const std::vector& IntervalList() const { return _intervals; } private: friend class Strategy; /** @brief It is not allowed to copy this class */ AOFlagger(const AOFlagger&) = delete; /** @brief It is not allowed to assign to this class */ void operator=(const AOFlagger&) = delete; StatusListener* _statusListener; std::vector _antennas; std::vector _bands; std::vector _intervals; }; } // namespace aoflagger #endif aoflagger-v3.5.1/interface/aoflagger.cpp0000664000175000017500000000255314752462134016367 0ustar oleole#include "aoflagger.h" #include "../lua/telescopefile.h" #include namespace aoflagger { std::string AOFlagger::FindStrategyFile(enum TelescopeId telescopeId, const std::string& scenario) { return TelescopeFile::FindStrategy( static_cast(telescopeId), scenario); } QualityStatistics AOFlagger::MakeQualityStatistics( const double* scanTimes, size_t nScans, const double* channelFrequencies, size_t nChannels, size_t nPolarizations) { return QualityStatistics(scanTimes, nScans, channelFrequencies, nChannels, nPolarizations, false); } QualityStatistics AOFlagger::MakeQualityStatistics( const double* scanTimes, size_t nScans, const double* channelFrequencies, size_t nChannels, size_t nPolarizations, bool computeHistograms) { return QualityStatistics(scanTimes, nScans, channelFrequencies, nChannels, nPolarizations, computeHistograms); } std::string AOFlagger::GetVersionString() { return AOFLAGGER_VERSION_STR; } void AOFlagger::GetVersion(short& major, short& minor, short& subMinor) { major = AOFLAGGER_VERSION_MAJOR; minor = AOFLAGGER_VERSION_MINOR; subMinor = AOFLAGGER_VERSION_SUBMINOR; } std::string AOFlagger::GetVersionDate() { return AOFLAGGER_VERSION_DATE_STR; } } // end of namespace aoflagger aoflagger-v3.5.1/interface/interface.dox0000664000175000017500000000165014752462134016405 0ustar oleole/** \mainpage AOFlagger public interface API * * ### About the API * * These pages describe the C++ API of the public interface. This interface can * be used to integrate the flagger into an observatory pipeline in an easy * way, without having to know the internals of the flagger. If this flagger * is to be integrated in a pipeline, the best place to start is to * read the class description of class @ref aoflagger::AOFlagger. * This interface has also been wrapped in a Python interface. * * To use libaoflagger, add an "#include " to your C++ * program and link with "-laoflagger". For installation instructions, check * the website. * * The official webpage of the full documentation is http://aoflagger.readthedocs.io/ . * Additionally, the * * LOFAR imaging cookbook contains a lot of general info about running the * flagger. */ aoflagger-v3.5.1/interface/imageset.cpp0000664000175000017500000000751414752462134016240 0ustar oleole#include "aoflagger.h" #include "structures.h" #include "../structures/image2d.h" #include namespace aoflagger { ImageSet::ImageSet() : _data(nullptr) {} ImageSet::ImageSet(size_t width, size_t height, size_t count) : _data(new ImageSetData(count)) { assertValidCount(count); for (size_t i = 0; i != count; ++i) _data->images[i] = Image2D::CreateUnsetImagePtr(width, height); } ImageSet::ImageSet(size_t width, size_t height, size_t count, float initialValue) : _data(new ImageSetData(count)) { assertValidCount(count); for (size_t i = 0; i != count; ++i) _data->images[i] = Image2D::CreateSetImagePtr(width, height, initialValue); } ImageSet::ImageSet(size_t width, size_t height, size_t count, size_t widthCapacity) : _data(new ImageSetData(count)) { assertValidCount(count); for (size_t i = 0; i != count; ++i) _data->images[i] = Image2D::CreateUnsetImagePtr(width, height, widthCapacity); } ImageSet::ImageSet(size_t width, size_t height, size_t count, float initialValue, size_t widthCapacity) : _data(new ImageSetData(count)) { assertValidCount(count); for (size_t i = 0; i != count; ++i) _data->images[i] = Image2D::CreateSetImagePtr(width, height, initialValue, widthCapacity); } ImageSet::ImageSet(const ImageSet& sourceImageSet) : _data(sourceImageSet._data != nullptr ? new ImageSetData(*sourceImageSet._data) : nullptr) {} ImageSet::ImageSet::ImageSet(aoflagger::ImageSet&& sourceImageSet) : _data(std::move(sourceImageSet._data)) {} ImageSet::~ImageSet() {} ImageSet& ImageSet::operator=(const ImageSet& sourceImageSet) { if (sourceImageSet._data == nullptr) { _data.reset(); } else if (_data == nullptr) { _data.reset(new ImageSetData(*sourceImageSet._data)); } else { *_data = *sourceImageSet._data; } return *this; } ImageSet& ImageSet::operator=(ImageSet&& sourceImageSet) { std::swap(_data, sourceImageSet._data); return *this; } void ImageSet::assertValidCount(size_t count) { if (count != 1 && count != 2 && count != 4 && count != 8) throw std::runtime_error( "Invalid count specified when creating image set for aoflagger; should " "be 1, 2, 4 or 8."); } float* ImageSet::ImageBuffer(size_t imageIndex) { return _data->images[imageIndex]->Data(); } const float* ImageSet::ImageBuffer(size_t imageIndex) const { return _data->images[imageIndex]->Data(); } size_t ImageSet::Width() const { return _data->images[0]->Width(); } size_t ImageSet::Height() const { return _data->images[0]->Height(); } size_t ImageSet::ImageCount() const { return _data->images.size(); } size_t ImageSet::HorizontalStride() const { return _data->images[0]->Stride(); } void ImageSet::Set(float newValue) { for (const Image2DPtr& image : _data->images) { image->SetAll(newValue); } } void ImageSet::ResizeWithoutReallocation(size_t newWidth) const { for (const Image2DPtr& image : _data->images) { image->ResizeWithoutReallocation(newWidth); } } void ImageSet::SetAntennas(size_t antenna1, size_t antenna2) { _data->hasAntennas = true; _data->antenna1 = antenna1; _data->antenna2 = antenna2; } bool ImageSet::HasAntennas() const { return _data->hasAntennas; } size_t ImageSet::Antenna1() const { return _data->antenna1; } size_t ImageSet::Antenna2() const { return _data->antenna2; } void ImageSet::SetInterval(size_t index) { _data->interval = index; _data->hasInterval = true; } bool ImageSet::HasInterval() const { return _data->hasInterval; } size_t ImageSet::Interval() const { return _data->interval; } void ImageSet::SetBand(size_t index) { _data->band = index; _data->hasBand = true; } bool ImageSet::HasBand() const { return _data->hasBand; } size_t ImageSet::Band() const { return _data->band; } } // namespace aoflagger aoflagger-v3.5.1/imaging/0000775000175000017500000000000015146315735013404 5ustar oleoleaoflagger-v3.5.1/imaging/model.cpp0000664000175000017500000002637615063016106015212 0ustar oleole#include "model.h" #include "../structures/image2d.h" #include "../structures/timefrequencydata.h" #include "observatorium.h" #include "../util/rng.h" #include "../util/logger.h" #include Model::Model() : _noiseSigma(1.0), _sourceSigma(0.0), _integrationTime(15.0) {} template void Model::SimulateObservation(struct OutputReceiver& receiver, Observatorium& observatorium, num_t delayDirectionDEC, num_t delayDirectionRA) { const size_t channelCount = observatorium.BandInfo().channels.size(); const double frequency = observatorium.BandInfo().channels[0].frequencyHz; for (size_t f = 0; f < channelCount; ++f) { const double channelFrequency = frequency + observatorium.ChannelWidthHz() * f; receiver.SetY(f); for (size_t i = 0; i < observatorium.AntennaCount(); ++i) { for (size_t j = i + 1; j < observatorium.AntennaCount(); ++j) { const AntennaInfo &antenna1 = observatorium.GetAntenna(i), &antenna2 = observatorium.GetAntenna(j); double dx = antenna1.position.x - antenna2.position.x, dy = antenna1.position.y - antenna2.position.y, dz = antenna1.position.z - antenna2.position.z; SimulateCorrelation(receiver, delayDirectionDEC, delayDirectionRA, dx, dy, dz, channelFrequency, observatorium.ChannelWidthHz(), 12 * 60 * 60, _integrationTime); } } } } template void Model::SimulateObservation( struct OutputReceiver& receiver, Observatorium& observatorium, num_t delayDirectionDEC, num_t delayDirectionRA); template void Model::SimulateObservation( struct OutputReceiver& receiver, Observatorium& observatorium, num_t delayDirectionDEC, num_t delayDirectionRA); std::pair Model::SimulateObservation(size_t nTimes, class Observatorium& observatorium, num_t delayDirectionDEC, num_t delayDirectionRA, size_t a1, size_t a2) { const size_t channelCount = observatorium.BandInfo().channels.size(); const double frequency = observatorium.BandInfo().channels[0].frequencyHz; OutputReceiver tfOutputter; tfOutputter._real = Image2D::CreateZeroImagePtr(nTimes, channelCount); tfOutputter._imaginary = Image2D::CreateZeroImagePtr(nTimes, channelCount); const TimeFrequencyMetaDataPtr metaData(new TimeFrequencyMetaData()); metaData->SetAntenna1(observatorium.GetAntenna(a1)); metaData->SetAntenna2(observatorium.GetAntenna(a2)); metaData->SetBand(observatorium.BandInfo()); double dx = metaData->Antenna1().position.x - metaData->Antenna2().position.x, dy = metaData->Antenna1().position.y - metaData->Antenna2().position.y, dz = metaData->Antenna1().position.z - metaData->Antenna2().position.z; for (size_t f = 0; f < channelCount; ++f) { const double channelFrequency = frequency + observatorium.ChannelWidthHz() * f; tfOutputter.SetY(f); SimulateCorrelation(tfOutputter, delayDirectionDEC, delayDirectionRA, dx, dy, dz, channelFrequency, observatorium.ChannelWidthHz(), nTimes, _integrationTime); } std::vector times; std::vector uvws; const num_t wavelength = 1.0L / frequency; for (size_t i = 0; i != nTimes; ++i) { const double t = _integrationTime * i; times.push_back(t); const num_t earthLattitudeApprox = t * M_PIn / (12.0 * 60.0 * 60.0); UVW uvw; GetUVPosition(uvw.u, uvw.v, earthLattitudeApprox, delayDirectionDEC, delayDirectionRA, dx, dy, dz, wavelength); uvw.u = uvw.u * (299792458.0L / frequency); uvw.v = uvw.v * (299792458.0L / frequency); uvw.w = GetWPosition(delayDirectionDEC, delayDirectionRA, frequency, earthLattitudeApprox, dx, dy) * (299792458.0L / frequency); uvws.push_back(uvw); } metaData->SetUVW(uvws); metaData->SetObservationTimes(times); FieldInfo field; field.fieldId = 0; field.delayDirectionDec = delayDirectionDEC; // field.delayDirectionDecNegCos = -cos(delayDirectionDEC); // field.delayDirectionDecNegSin = -sin(delayDirectionDEC); field.delayDirectionRA = delayDirectionRA; metaData->SetField(field); const TimeFrequencyData tfData(aocommon::Polarization::StokesI, tfOutputter._real, tfOutputter._imaginary); return std::pair(tfData, metaData); } template void Model::SimulateCorrelation(struct OutputReceiver& receiver, num_t delayDirectionDEC, num_t delayDirectionRA, num_t dx, num_t dy, num_t dz, num_t frequency, num_t channelWidth, size_t nTimes, double integrationTime) { const double sampleGain = integrationTime / (12.0 * 60.0 * 60.0) * channelWidth; const num_t wavelength = 1.0L / frequency; size_t index = 0; for (size_t ti = 0; ti != nTimes; ++ti) { const double t = ti * integrationTime; const double timeInDays = t / (12.0 * 60.0 * 60.0); const num_t earthLattitudeApprox = timeInDays * M_PIn; num_t u, v, r1, i1, r2, i2; GetUVPosition(u, v, earthLattitudeApprox, delayDirectionDEC, delayDirectionRA, dx, dy, dz, wavelength); SimulateAntenna(timeInDays, delayDirectionDEC, delayDirectionRA, 0, 0, frequency, earthLattitudeApprox, r1, i1); SimulateAntenna(timeInDays, delayDirectionDEC, delayDirectionRA, dx, dy, frequency, earthLattitudeApprox, r2, i2); num_t r = r1 * r2 - (i1 * -i2), i = r1 * -i2 + r2 * i1; receiver.SetUVValue(index, u, v, r * sampleGain, i * sampleGain, 1.0); ++index; } } template void Model::SimulateCorrelation( struct OutputReceiver& receiver, num_t delayDirectionDEC, num_t delayDirectionRA, num_t dx, num_t dy, num_t dz, num_t frequency, num_t channelWidth, size_t nTimes, double integrationTime); void Model::SimulateAntenna(double time, num_t delayDirectionDEC, num_t delayDirectionRA, num_t dx, num_t dy, num_t frequency, num_t earthLattitude, num_t& r, num_t& i) { r = 0.0; i = 0.0; const num_t delayW = GetWPosition(delayDirectionDEC, delayDirectionRA, frequency, earthLattitude, dx, dy); for (std::vector::const_iterator iter = _sources.begin(); iter != _sources.end(); ++iter) { const Source& source = **iter; const num_t w = GetWPosition(source.Dec(time), source.Ra(time), frequency, earthLattitude, dx, dy); const num_t fieldStrength = source.SqrtFluxIntensity(time) + RNG::Gaussian() * _sourceSigma; num_t noiser, noisei; RNG::ComplexGaussianAmplitude(noiser, noisei); r += fieldStrength * std::cos((w - delayW) * M_PIn * 2.0) + noiser * _noiseSigma; i += fieldStrength * std::sin((w - delayW) * M_PIn * 2.0) + noisei * _noiseSigma; } } void Model::SimulateUncoherentAntenna(double time, num_t delayDirectionDEC, num_t delayDirectionRA, num_t dx, num_t dy, num_t frequency, num_t earthLattitude, num_t& r, num_t& i, size_t index) { const num_t delayW = GetWPosition(delayDirectionDEC, delayDirectionRA, frequency, earthLattitude, dx, dy); // if(index%(_sources.size()+1) == _sources.size()) //{ num_t noiser, noisei; RNG::ComplexGaussianAmplitude(noiser, noisei); noiser *= _noiseSigma; noisei *= _noiseSigma; //} // else { const Source& source = *_sources[index % _sources.size()]; const num_t w = GetWPosition(source.Dec(time), source.Ra(time), frequency, earthLattitude, dx, dy); const num_t fieldStrength = source.SqrtFluxIntensity(time) + RNG::Gaussian() * _sourceSigma; r = fieldStrength * std::cos((w - delayW) * M_PIn * 2.0) + noiser; i = fieldStrength * std::sin((w - delayW) * M_PIn * 2.0) + noisei; //} } void Model::GetUVPosition(num_t& u, num_t& v, num_t earthLattitudeAngle, num_t delayDirectionDEC, num_t delayDirectionRA, num_t dx, num_t dy, num_t dz, num_t wavelength) { // Rotate baseline plane towards phase center, first rotate around z axis, // then around x axis const long double raRotation = -earthLattitudeAngle + delayDirectionRA + M_PIn * 0.5L; long double tmpCos = std::cos(raRotation); long double tmpSin = std::sin(raRotation); const long double dxProjected = tmpCos * dx - tmpSin * dy; const long double tmpdy = tmpSin * dx + tmpCos * dy; tmpCos = std::cos(-delayDirectionDEC); tmpSin = std::sin(-delayDirectionDEC); const long double dyProjected = tmpCos * tmpdy - tmpSin * dz; // Now, the newly projected positive z axis of the baseline points to the // phase center long double baselineLength = std::sqrt(dxProjected * dxProjected + dyProjected * dyProjected); long double baselineAngle; if (baselineLength == 0.0) { baselineAngle = 0.0; } else { baselineLength /= 299792458.0L * wavelength; if (dxProjected > 0.0L) baselineAngle = std::atan(dyProjected / dxProjected); else baselineAngle = M_PIn - std::atan(dyProjected / -dxProjected); } u = std::cos(baselineAngle) * baselineLength; v = -std::sin(baselineAngle) * baselineLength; } void Model::loadUrsaMajor(double ra, double dec, double factor) { double s = 0.00005 * factor, // scale rs = 6.0 + 2.0 * factor; // stretch in dec const double fluxoffset = 0.0; AddSource(dec + s * rs * 40, ra + s * 72, 8.0 / 8.0 + fluxoffset); // Dubhe AddSource(dec + s * rs * -16, ra + s * 81, 4.0 / 8.0 + fluxoffset); // Beta AddSource(dec + s * rs * -45, ra + s * 2, 3.0 / 8.0 + fluxoffset); // Gamma AddSource(dec + s * rs * -6, ra + s * -27, 2.0 / 8.0 + fluxoffset); // Delta AddSource(dec + s * rs * -4, ra + s * -85, 6.0 / 8.0 + fluxoffset); // Alioth AddSource(dec + s * rs * 2, ra + s * -131, 5.0 / 8.0 + fluxoffset); // Zeta AddSource(dec + s * rs * -36, ra + s * -192, 7.0 / 8.0 + fluxoffset); // Alkaid // AddSource(cd, cr - M_PI, 4.0); } void Model::loadUrsaMajorDistortingSource(double ra, double dec, double factor, bool slightlyMiss) { if (slightlyMiss) { dec += 0.005; ra += 0.002; } AddSource(dec - 0.12800 * factor, ra + 0.015 + 0.015 * factor, 4.0); } void Model::loadUrsaMajorDistortingVariableSource(double ra, double dec, double factor, bool weak, bool slightlyMiss) { double flux = 4.0; dec = dec - 0.12800 * factor; ra = ra + 0.015 + 0.015 * factor; if (slightlyMiss) { dec += 0.005; ra += 0.002; } if (weak) { flux /= 100.0; } AddVariableSource(dec, ra, flux); } aoflagger-v3.5.1/imaging/observatorium.h0000664000175000017500000001202314752462134016452 0ustar oleole#ifndef OBSERVATORIUM_H #define OBSERVATORIUM_H #include #include "../structures/antennainfo.h" class Observatorium { public: Observatorium() : _channelWidthHz(0.0) {} void AddAntenna(AntennaInfo& antenna) { _antennae.push_back(antenna); } size_t AntennaCount() const { return _antennae.size(); } const AntennaInfo& GetAntenna(size_t index) const { return _antennae[index]; } void SetChannelWidthHz(double channelWidthHz) { _channelWidthHz = channelWidthHz; } double ChannelWidthHz() const { return _channelWidthHz; } const class BandInfo& BandInfo() const { return _bandInfo; } protected: class BandInfo& GetBandInfo() { return _bandInfo; } private: std::vector _antennae; double _channelWidthHz; class BandInfo _bandInfo; }; struct WSRTObservatorium : public Observatorium { explicit WSRTObservatorium(size_t channelCount = 16 * 4, double bandwidthHz = 2500000.0 * 16.0) { AntennaInfo antennas[14]; for (unsigned i = 0; i < 14; ++i) { WSRTCommon(antennas[i]); WSRTn(i, antennas[i]); AddAntenna(antennas[i]); } SetChannelWidthHz(bandwidthHz / channelCount); initBand(channelCount); } explicit WSRTObservatorium(size_t antenna1, size_t antenna2, size_t channelCount = 16 * 4) { AntennaInfo antennas[2]; WSRTCommon(antennas[0]); WSRTCommon(antennas[1]); WSRTn(antenna1, antennas[0]); WSRTn(antenna2, antennas[1]); AddAntenna(antennas[0]); AddAntenna(antennas[1]); SetChannelWidthHz(10000.0 * 256.0 * 16.0 / channelCount); initBand(channelCount); } private: void WSRTCommon(AntennaInfo& antenna) { antenna.diameter = 25; antenna.mount = "equatorial"; antenna.station = "WSRT"; } void WSRTn(size_t antennaIndex, AntennaInfo& antenna) { switch (antennaIndex) { case 0: antenna.id = 0; antenna.name = "RT0"; antenna.position.x = 3.82876e+06; antenna.position.y = 442449; antenna.position.z = 5.06492e+06; break; case 1: antenna.id = 1; antenna.name = "RT1"; antenna.position.x = 3.82875e+06; antenna.position.y = 442592; antenna.position.z = 5.06492e+06; break; case 2: antenna.id = 2; antenna.name = "RT2"; antenna.position.x = 3.82873e+06; antenna.position.y = 442735; antenna.position.z = 5.06492e+06; break; case 3: antenna.id = 3; antenna.name = "RT3"; antenna.position.x = 3.82871e+06; antenna.position.y = 442878; antenna.position.z = 5.06492e+06; break; case 4: antenna.id = 4; antenna.name = "RT4"; antenna.position.x = 3.8287e+06; antenna.position.y = 443021; antenna.position.z = 5.06492e+06; break; case 5: antenna.id = 5; antenna.name = "RT5"; antenna.position.x = 3.82868e+06; antenna.position.y = 443164; antenna.position.z = 5.06492e+06; break; case 6: antenna.id = 6; antenna.name = "RT6"; antenna.position.x = 3.82866e+06; antenna.position.y = 443307; antenna.position.z = 5.06492e+06; break; case 7: antenna.id = 7; antenna.name = "RT7"; antenna.position.x = 3.82865e+06; antenna.position.y = 443450; antenna.position.z = 5.06492e+06; break; case 8: antenna.id = 8; antenna.name = "RT8"; antenna.position.x = 3.82863e+06; antenna.position.y = 443593; antenna.position.z = 5.06492e+06; break; case 9: antenna.id = 9; antenna.name = "RT9"; antenna.position.x = 3.82861e+06; antenna.position.y = 443736; antenna.position.z = 5.06492e+06; break; case 10: antenna.id = 10; antenna.name = "RTA"; antenna.position.x = 3.8286e+06; antenna.position.y = 443832; antenna.position.z = 5.06492e+06; break; case 11: antenna.id = 11; antenna.name = "RTB"; antenna.position.x = 3.82859e+06; antenna.position.y = 443903; antenna.position.z = 5.06492e+06; break; case 12: antenna.id = 12; antenna.name = "RTC"; antenna.position.x = 3.82845e+06; antenna.position.y = 445119; antenna.position.z = 5.06492e+06; break; case 13: antenna.id = 13; antenna.name = "RTD"; antenna.position.x = 3.82845e+06; antenna.position.y = 445191; antenna.position.z = 5.06492e+06; break; } } void initBand(size_t channelCount) { GetBandInfo().windowIndex = 0; for (size_t i = 0; i < channelCount; ++i) { ChannelInfo channel; channel.frequencyIndex = i; channel.channelWidthHz = ChannelWidthHz(); channel.frequencyHz = 147000000.0 - 20000000.0 + ChannelWidthHz() * i; GetBandInfo().channels.push_back(channel); } } }; #endif aoflagger-v3.5.1/imaging/uvimager.h0000664000175000017500000001624615063016106015371 0ustar oleole#ifndef UVIMAGER_H #define UVIMAGER_H #include #include "../structures/timefrequencymetadata.h" #include "../structures/date.h" #include "../structures/timefrequencydata.h" struct SingleFrequencySingleBaselineData { std::complex data; bool flag; bool available; double time; unsigned field; }; class UVImager { public: enum ImageKind { Homogeneous, Flagging }; UVImager(unsigned long xRes, unsigned long yRes, ImageKind imageKind = Homogeneous); ~UVImager(); void Image(class MSMetaData& msMetaData, unsigned band); void Image(class MSMetaData& msMetaData, unsigned band, const class IntegerDomain& frequencies); void Image(const class TimeFrequencyData& data, TimeFrequencyMetaDataCPtr metaData, unsigned frequencyIndex); void Image(const class TimeFrequencyData& data, TimeFrequencyMetaDataCPtr metaData) { for (unsigned y = 0; y < data.ImageHeight(); ++y) Image(data, metaData, y); } void Image(const class TimeFrequencyData& data, class SpatialMatrixMetaData* metaData); void InverseImage(class MSMetaData& prototype, unsigned band, const class Image2D& uvReal, const class Image2D& uvImaginary, unsigned antenna1, unsigned antenna2); const class Image2D& WeightImage() const { return _uvWeights; } const class Image2D& RealUVImage() const { return _uvReal; } const class Image2D& ImaginaryUVImage() const { return _uvImaginary; } void SetInvertFlagging(bool newValue) { _invertFlagging = newValue; } void SetDirectFT(bool directFT) { _directFT = directFT; } /** * This function calculates the uv position, but it's not optimized for speed, * so it's not to be used in an imager. * @param [out] u the u position (in the uv-plane domain) * @param [out] v the v position (in the uv-plane domain) * @param [in] timeIndex the time index to calculate the u,v position for * @param [in] frequencyIndex the frequency index to calculate the u,v * position for * @param [in] metaData information about the baseline */ static void GetUVPosition(num_t& u, num_t& v, size_t timeIndex, size_t frequencyIndex, TimeFrequencyMetaDataCPtr metaData); static num_t GetFringeStopFrequency(size_t time, const Baseline& baseline, num_t delayDirectionRA, num_t delayDirectionDec, num_t frequency, TimeFrequencyMetaDataCPtr metaData); // static double GetFringeCount(long double timeStart, long double timeEnd, // const Baseline &baseline, long double delayDirectionRA, long double // delayDirectionDec, long double frequency); static num_t GetFringeCount(size_t timeIndexStart, size_t timeIndexEnd, unsigned channelIndex, TimeFrequencyMetaDataCPtr metaData); static numl_t GetWPosition(numl_t delayDirectionDec, numl_t delayDirectionRA, numl_t frequency, numl_t earthLattitudeAngle, numl_t dx, numl_t dy) { numl_t wavelength = 299792458.0L / frequency; numl_t raSinEnd = std::sin(-delayDirectionRA - earthLattitudeAngle); numl_t raCosEnd = std::cos(-delayDirectionRA - earthLattitudeAngle); numl_t decCos = std::cos(delayDirectionDec); // term "+ dz * decCos" is eliminated because of subtraction num_t wPosition = (dx * raCosEnd - dy * raSinEnd) * (-decCos) / wavelength; return wPosition; } static numl_t TimeToEarthLattitude(unsigned x, TimeFrequencyMetaDataCPtr metaData) { return TimeToEarthLattitude(metaData->ObservationTimes()[x]); } static numl_t TimeToEarthLattitude(double time) { return time * M_PInl / (12.0 * 60.0 * 60.0); } void Empty(); void PerformFFT(); bool HasUV() const { return !_uvReal.Empty(); } bool HasFFT() const { return !_uvFTReal.Empty(); } const class Image2D& FTReal() const { return _uvFTReal; } const class Image2D& FTImaginary() const { return _uvFTImaginary; } class Image2D& FTReal() { return _uvFTReal; } class Image2D& FTImaginary() { return _uvFTImaginary; } void SetUVScaling(num_t newScale) { _uvScaling = newScale; } num_t UVScaling() const { return _uvScaling; } void ApplyWeightsToUV(); void SetUVValue(num_t u, num_t v, num_t r, num_t i, num_t weight); template static T FrequencyToWavelength(const T frequency) { return SpeedOfLight() / frequency; } static long double SpeedOfLight() { return 299792458.0L; } numl_t ImageDistanceToDecRaDistance(numl_t imageDistance) const { return imageDistance * _uvScaling; } static numl_t AverageUVDistance(TimeFrequencyMetaDataCPtr metaData, const double frequencyHz) { const std::vector& uvw = metaData->UVW(); numl_t avgDist = 0.0; for (std::vector::const_iterator i = uvw.begin(); i != uvw.end(); ++i) { numl_t dist = i->u * i->u + i->v * i->v; avgDist += std::sqrt(dist); } return avgDist * frequencyHz / (SpeedOfLight() * (numl_t)uvw.size()); } static numl_t UVTrackLength(TimeFrequencyMetaDataCPtr metaData, const double frequencyHz) { const std::vector& uvw = metaData->UVW(); numl_t length = 0.0; std::vector::const_iterator i = uvw.begin(); if (i == uvw.end()) return 0.0; while ((i + 1) != uvw.end()) { std::vector::const_iterator n = i; ++n; const numl_t du = n->u - i->u, dv = n->v - i->v; length += std::sqrt(du * du + dv * dv); i = n; } return length * frequencyHz / SpeedOfLight(); } numl_t ImageDistanceToFringeSpeedInSamples( numl_t imageDistance, double frequencyHz, TimeFrequencyMetaDataCPtr metaData) const { return ImageDistanceToDecRaDistance(imageDistance) * AverageUVDistance(metaData, frequencyHz) / (0.5 * (numl_t)metaData->UVW().size()); } private: void Clear(); struct AntennaCache { num_t wavelength; num_t dx, dy, dz; }; void Image(const class IntegerDomain& frequencies); void Image(const IntegerDomain& frequencies, const IntegerDomain& antenna1Domain, const IntegerDomain& antenna2Domain); void Image(unsigned frequencyIndex, class AntennaInfo& antenna1, class AntennaInfo& antenna2, SingleFrequencySingleBaselineData* data); // This is the fast variant. void GetUVPosition(num_t& u, num_t& v, const SingleFrequencySingleBaselineData& data, const AntennaCache& cache); void SetUVFTValue(num_t u, num_t v, num_t r, num_t i, num_t weight); unsigned long _xRes, _yRes; unsigned long _xResFT, _yResFT; num_t _uvScaling; class Image2D _uvReal, _uvImaginary, _uvWeights; class Image2D _uvFTReal, _uvFTImaginary; class Image2D _timeFreq; MSMetaData* _msMetaData; unsigned _antennaCount, _fieldCount; AntennaInfo* _antennas; BandInfo _band; FieldInfo* _fields; size_t _scanCount; ImageKind _imageKind; bool _invertFlagging, _directFT; bool _ignoreBoundWarnings; }; #endif aoflagger-v3.5.1/imaging/defaultmodels.h0000664000175000017500000000373214752462134016410 0ustar oleole#ifndef DEFAULTMODELS_H #define DEFAULTMODELS_H #include #include "model.h" #include "observatorium.h" enum class SourceSet { NoDistortion, ConstantDistortion, StrongVariableDistortion, FaintVariableDistortion }; class DefaultModels { public: enum SetLocation { EmptySet, NCPSet }; static double DistortionRA() { return 4.940; } static double DistortionDec() { return 0.571; } static std::pair LoadSet( enum SetLocation setLocation, enum SourceSet distortion, double noiseSigma, size_t channelCount = 64, double bandwidth = 2500000.0 * 16.0, unsigned a1 = 0, unsigned a2 = 5) { double ra, dec, factor; getSetData(setLocation, ra, dec, factor); Model model; model.SetNoiseSigma(noiseSigma); if (setLocation != EmptySet) model.loadUrsaMajor(ra, dec, factor); switch (distortion) { case SourceSet::NoDistortion: break; case SourceSet::ConstantDistortion: model.loadUrsaMajorDistortingSource(ra, dec, factor, true); break; case SourceSet::StrongVariableDistortion: model.loadUrsaMajorDistortingVariableSource(ra, dec, factor, false, false); break; case SourceSet::FaintVariableDistortion: model.loadUrsaMajorDistortingVariableSource(ra, dec, factor, true, false); break; } WSRTObservatorium wsrtObservatorium(channelCount, bandwidth); return model.SimulateObservation(12 * 60 * 60 / 15, wsrtObservatorium, dec, ra, a1, a2); } private: static void getSetData(enum SetLocation setLocation, double& ra, double& dec, double& factor) { if (setLocation == NCPSet) { dec = 0.5 * M_PI + 0.12800; ra = -0.03000; factor = 1.0; } else { dec = 1.083; ra = 4.865; factor = 4.0; } } }; #endif aoflagger-v3.5.1/imaging/model.h0000664000175000017500000001304215063016106014641 0ustar oleole#ifndef MODEL_H #define MODEL_H #include #include #include #include "../structures/image2d.h" #include "../structures/types.h" #include "uvimager.h" template struct OutputReceiver { void SetY(size_t) {} }; template <> struct OutputReceiver { UVImager* _imager; void SetUVValue(size_t, double u, double v, double r, double i, double w) { _imager->SetUVValue(u, v, r, i, w); _imager->SetUVValue(-u, -v, r, -i, w); } void SetY(size_t) {} }; template <> struct OutputReceiver { Image2DPtr _real, _imaginary; size_t _y; void SetUVValue(size_t x, double, double, double r, double i, double) { _real->SetValue(x, _y, r); _imaginary->SetValue(x, _y, i); } void SetY(size_t y) { _y = y; } }; class Model { struct Source { virtual ~Source() {} virtual numl_t Dec(num_t t) const = 0; virtual numl_t Ra(num_t t) const = 0; virtual numl_t FluxIntensity(num_t t) const = 0; virtual numl_t SqrtFluxIntensity(num_t t) const { return std::sqrt(FluxIntensity(t)); } }; struct StablePointSource final : public Source { long double dec, ra, fluxIntensity, sqrtFluxIntensity; numl_t Dec(num_t) const override { return dec; } numl_t Ra(num_t) const override { return ra; } numl_t FluxIntensity(num_t) const override { return fluxIntensity; } numl_t SqrtFluxIntensity(num_t) const override { return sqrtFluxIntensity; } }; struct VariablePointSource final : public Source { long double dec, ra, fluxIntensity; double peakTime, oneOverSigmaSq; numl_t Dec(num_t) const override { return dec; } numl_t Ra(num_t) const override { return ra; } numl_t FluxIntensity(num_t t) const override { numl_t mu = std::fmod(std::fabs(t - peakTime), 1.0); if (mu > 0.5) mu = 1.0 - mu; return fluxIntensity * (1.0 + std::exp(mu * mu * oneOverSigmaSq)) * (1.0 + std::fmod(t * 1007.0, 13.0) / 26.0); } }; public: Model(); void AddSource(long double dec, long double ra, long double fluxIntensity) { StablePointSource* source = new StablePointSource(); source->dec = dec; source->ra = ra; source->fluxIntensity = fluxIntensity; source->sqrtFluxIntensity = sqrt(fluxIntensity); _sources.push_back(source); } void AddVariableSource(long double dec, long double ra, long double fluxIntensity) { VariablePointSource* source = new VariablePointSource(); source->dec = dec; source->ra = ra; source->fluxIntensity = fluxIntensity; source->peakTime = 0.2; source->oneOverSigmaSq = 1.0 / (0.3 * 0.3); _sources.push_back(source); } void SimulateAntenna(double time, num_t delayDirectionDEC, num_t delayDirectionRA, num_t dx, num_t dy, num_t frequency, num_t earthLattitude, num_t& r, num_t& i); void SimulateUncoherentAntenna(double time, num_t delayDirectionDEC, num_t delayDirectionRA, num_t dx, num_t dy, num_t frequency, num_t earthLattitude, num_t& r, num_t& i, size_t index); template void SimulateCorrelation(struct OutputReceiver& receiver, num_t delayDirectionDEC, num_t delayDirectionRA, num_t dx, num_t dy, num_t dz, num_t frequency, num_t channelWidth, size_t nTimes, double integrationTime); void SimulateObservation(class UVImager& imager, class Observatorium& observatorium, num_t delayDirectionDEC, num_t delayDirectionRA) { srand(1); OutputReceiver imagerOutputter; imagerOutputter._imager = &imager; SimulateObservation(imagerOutputter, observatorium, delayDirectionDEC, delayDirectionRA); } std::pair SimulateObservation( size_t nTimes, class Observatorium& observatorium, num_t delayDirectionDEC, num_t delayDirectionRA, size_t a1, size_t a2); template void SimulateObservation(struct OutputReceiver& receiver, class Observatorium& observatorium, num_t delayDirectionDEC, num_t delayDirectionRA); static void GetUVPosition(num_t& u, num_t& v, num_t earthLattitudeAngle, num_t delayDirectionDEC, num_t delayDirectionRA, num_t dx, num_t dy, num_t dz, num_t waveLength); static num_t GetWPosition(num_t delayDirectionDec, num_t delayDirectionRA, num_t frequency, num_t earthLattitudeAngle, num_t dx, num_t dy) { return UVImager::GetWPosition(delayDirectionDec, delayDirectionRA, frequency, earthLattitudeAngle, dx, dy); } void loadUrsaMajor(double ra, double dec, double factor); void loadUrsaMajorDistortingSource(double ra, double dec, double factor, bool slightlyMiss = false); void loadUrsaMajorDistortingVariableSource(double ra, double dec, double factor, bool weak = false, bool slightlyMiss = false); double NoiseSigma() const { return _noiseSigma; } void SetNoiseSigma(double noiseSigma) { _noiseSigma = noiseSigma; } private: std::vector _sources; double _noiseSigma, _sourceSigma; double _integrationTime; }; #endif aoflagger-v3.5.1/imaging/uvimager.cpp0000664000175000017500000004566015063016106015726 0ustar oleole#include "uvimager.h" #include "../structures/image2d.h" #include "../structures/mask2d.h" #include "../structures/msmetadata.h" #include "../structures/spatialmatrixmetadata.h" #include "../structures/timefrequencydata.h" #include "../util/integerdomain.h" #include "../util/stopwatch.h" #include "../util/ffttools.h" #include #include #include UVImager::UVImager(unsigned long xRes, unsigned long yRes, ImageKind imageKind) : _xRes(xRes), _yRes(yRes), _xResFT(xRes), _yResFT(yRes), _uvReal(), _uvImaginary(), _uvWeights(), _uvFTReal(), _uvFTImaginary(), _antennas(nullptr), _fields(nullptr), _imageKind(imageKind), _invertFlagging(false), _directFT(false), _ignoreBoundWarnings(false) { _uvScaling = 0.0001L; // testing Empty(); } UVImager::~UVImager() { Clear(); } void UVImager::Clear() { if (_antennas != nullptr) { delete[] _antennas; _antennas = nullptr; } if (_fields != nullptr) { delete[] _fields; _fields = nullptr; } } void UVImager::Empty() { Clear(); _uvReal = Image2D::MakeZeroImage(_xRes, _yRes); _uvImaginary = Image2D::MakeZeroImage(_xRes, _yRes); _uvWeights = Image2D::MakeZeroImage(_xRes, _yRes); _uvFTReal = Image2D::MakeZeroImage(_xRes, _yRes); _uvFTImaginary = Image2D::MakeZeroImage(_xRes, _yRes); } void UVImager::Image(MSMetaData& msMetaData, unsigned band) { const unsigned frequencyCount = msMetaData.FrequencyCount(band); const IntegerDomain frequencies(0, frequencyCount); _msMetaData = &msMetaData; _band = _msMetaData->GetBandInfo(band); Image(frequencies); } void UVImager::Image(class MSMetaData& msMetaData, unsigned band, const IntegerDomain& frequencies) { _msMetaData = &msMetaData; _band = _msMetaData->GetBandInfo(band); Image(frequencies); } void UVImager::Image(const IntegerDomain& frequencies) { Empty(); _antennaCount = _msMetaData->AntennaCount(); _antennas = new AntennaInfo[_antennaCount]; for (unsigned i = 0; i < _antennaCount; ++i) _antennas[i] = _msMetaData->GetAntennaInfo(i); _fieldCount = _msMetaData->FieldCount(); _fields = new FieldInfo[_fieldCount]; for (unsigned i = 0; i < _fieldCount; ++i) _fields[i] = _msMetaData->GetFieldInfo(i); const unsigned parts = (frequencies.ValueCount() - 1) / 48 + 1; for (unsigned i = 0; i < parts; ++i) { std::cout << "Imaging " << i << "/" << parts << ":" << frequencies.Split(parts, i).ValueCount() << " frequencies..." << std::endl; Image(frequencies.Split(parts, i), IntegerDomain(0, _antennaCount), IntegerDomain(0, _antennaCount)); } } /** * Add several frequency channels to the uv plane for several combinations * of antenna pairs. */ void UVImager::Image(const IntegerDomain& frequencies, const IntegerDomain& antenna1Domain, const IntegerDomain& antenna2Domain) { _scanCount = _msMetaData->TimestepCount(); std::cout << "Requesting " << frequencies.ValueCount() << " x " << antenna1Domain.ValueCount() << " x " << antenna2Domain.ValueCount() << " x " << _scanCount << " x " << sizeof(SingleFrequencySingleBaselineData) << " = " << (frequencies.ValueCount() * antenna1Domain.ValueCount() * antenna2Domain.ValueCount() * _scanCount * sizeof(SingleFrequencySingleBaselineData) / (1024 * 1024)) << "MB of memory..." << std::endl; SingleFrequencySingleBaselineData**** data = new SingleFrequencySingleBaselineData***[frequencies.ValueCount()]; for (unsigned f = 0; f < frequencies.ValueCount(); ++f) { data[f] = new SingleFrequencySingleBaselineData**[antenna1Domain.ValueCount()]; for (unsigned a1 = 0; a1 < antenna1Domain.ValueCount(); ++a1) { data[f][a1] = new SingleFrequencySingleBaselineData*[antenna2Domain.ValueCount()]; for (unsigned a2 = 0; a2 < antenna2Domain.ValueCount(); ++a2) { data[f][a1][a2] = new SingleFrequencySingleBaselineData[_scanCount]; for (unsigned scan = 0; scan < _scanCount; ++scan) { data[f][a1][a2][scan].flag = true; data[f][a1][a2][scan].available = false; } } } } std::cout << "Reading all data for " << frequencies.ValueCount() << " frequencies..." << std::flush; Stopwatch stopwatch(true); const casacore::MeasurementSet ms(_msMetaData->Path()); const casacore::ScalarColumn antenna1Col( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::ANTENNA1)); const casacore::ScalarColumn antenna2Col( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::ANTENNA2)); const casacore::ScalarColumn fieldIdCol( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::FIELD_ID)); const casacore::ScalarColumn dataDescIdCol( ms, casacore::MeasurementSet::columnName( casacore::MeasurementSet::DATA_DESC_ID)); const casacore::ScalarColumn timeCol( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::TIME)); const casacore::ScalarColumn scanNumberCol( ms, casacore::MeasurementSet::columnName( casacore::MeasurementSet::SCAN_NUMBER)); const casacore::ArrayColumn correctedDataCol( ms, casacore::MeasurementSet::columnName( casacore::MeasurementSet::CORRECTED_DATA)); const casacore::ArrayColumn flagCol( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::FLAG)); const size_t rows = ms.nrow(); for (unsigned row = 0; row != rows; ++row) { const unsigned a1 = antenna1Col(row); const unsigned a2 = antenna2Col(row); if (antenna1Domain.IsIn(a1) && antenna2Domain.IsIn(a2)) { const unsigned scan = scanNumberCol(row); const unsigned index1 = antenna1Domain.Index(a1); const unsigned index2 = antenna1Domain.Index(a2); const int field = fieldIdCol(row); const double time = timeCol(row); casacore::Array dataArr = correctedDataCol(row); casacore::Array flagArr = flagCol(row); casacore::Array::const_iterator cdI = dataArr.begin(); casacore::Array::const_iterator fI = flagArr.begin(); for (int f = 0; f < frequencies.GetValue(0); ++f) { ++fI; ++fI; ++fI; ++fI; ++cdI; ++cdI; ++cdI; ++cdI; } for (unsigned f = 0; f < frequencies.ValueCount(); ++f) { SingleFrequencySingleBaselineData& curData = data[f][index1][index2][scan]; const casacore::Complex xxData = *cdI; ++cdI; ++cdI; ++cdI; const casacore::Complex yyData = *cdI; ++cdI; curData.data = xxData + yyData; bool flagging = *fI; ++fI; ++fI; ++fI; flagging = flagging || *fI; ++fI; curData.flag = flagging; curData.field = field; curData.time = time; curData.available = true; } } } stopwatch.Pause(); std::cout << "DONE in " << stopwatch.ToString() << " (" << (stopwatch.Seconds() / (antenna1Domain.ValueCount() * antenna1Domain.ValueCount())) << "s/antenna)" << std::endl; std::cout << "Imaging..." << std::flush; stopwatch.Reset(); stopwatch.Start(); for (unsigned f = 0; f < frequencies.ValueCount(); ++f) { for (unsigned a1 = 0; a1 < antenna1Domain.ValueCount(); ++a1) { for (unsigned a2 = 0; a2 < antenna2Domain.ValueCount(); ++a2) { Image(frequencies.GetValue(f), _antennas[antenna1Domain.GetValue(a1)], _antennas[antenna2Domain.GetValue(a2)], data[f][a1][a2]); } } } stopwatch.Pause(); std::cout << "DONE in " << stopwatch.ToString() << " (" << (stopwatch.Seconds() / (antenna1Domain.ValueCount() * antenna1Domain.ValueCount())) << "s/antenna)" << std::endl; // free data for (unsigned f = 0; f < frequencies.ValueCount(); ++f) { for (unsigned a1 = 0; a1 < antenna1Domain.ValueCount(); ++a1) { for (unsigned a2 = 0; a2 < antenna2Domain.ValueCount(); ++a2) { delete[] data[f][a1][a2]; } delete[] data[f][a1]; } delete[] data[f]; } delete[] data; } void UVImager::Image(unsigned frequencyIndex, AntennaInfo& antenna1, AntennaInfo& antenna2, SingleFrequencySingleBaselineData* data) { const num_t frequency = _band.channels[frequencyIndex].frequencyHz; const num_t speedOfLight = 299792458.0L; AntennaCache cache; cache.wavelength = speedOfLight / frequency; // dx, dy, dz is the baseline cache.dx = antenna1.position.x - antenna2.position.x; cache.dy = antenna1.position.y - antenna2.position.y; cache.dz = antenna1.position.z - antenna2.position.z; for (unsigned i = 0; i < _scanCount; ++i) { if (data[i].available) { switch (_imageKind) { case Homogeneous: if (!data[i].flag) { num_t u, v; GetUVPosition(u, v, data[i], cache); SetUVValue(u, v, data[i].data.real(), data[i].data.imag(), 1.0); SetUVValue(-u, -v, data[i].data.real(), -data[i].data.imag(), 1.0); // calcTimer.Pause(); } break; case Flagging: if ((data[i].flag && !_invertFlagging) || (!data[i].flag && _invertFlagging)) { num_t u, v; GetUVPosition(u, v, data[i], cache); SetUVValue(u, v, 1, 0, 1.0); SetUVValue(-u, -v, 1, 0, 1.0); } break; } } } } void UVImager::Image(const class TimeFrequencyData& data, class SpatialMatrixMetaData* metaData) { if (!_uvReal.Empty()) Empty(); Image2DCPtr real = data.GetRealPart(), imaginary = data.GetImaginaryPart(); const Mask2DCPtr flags = data.GetSingleMask(); for (unsigned a2 = 0; a2 < data.ImageHeight(); ++a2) { for (unsigned a1 = a2 + 1; a1 < data.ImageWidth(); ++a1) { num_t vr = real->Value(a1, a2), vi = imaginary->Value(a1, a2); if (std::isfinite(vr) && std::isfinite(vi)) { const UVW uvw = metaData->UVW(a1, a2); SetUVValue(uvw.u, uvw.v, vr, vi, 1.0); SetUVValue(-uvw.u, -uvw.v, vr, -vi, 1.0); } } } } void UVImager::Image(const TimeFrequencyData& data, TimeFrequencyMetaDataCPtr metaData, unsigned frequencyIndex) { if (!_uvReal.Empty()) Empty(); Image2DCPtr real = data.GetRealPart(), imaginary = data.GetImaginaryPart(); const Mask2DCPtr flags = data.GetSingleMask(); for (unsigned i = 0; i < data.ImageWidth(); ++i) { switch (_imageKind) { case Homogeneous: if (flags->Value(i, frequencyIndex) == 0.0L) { num_t vr = real->Value(i, frequencyIndex), vi = imaginary->Value(i, frequencyIndex); if (std::isfinite(vr) && std::isfinite(vi)) { num_t u, v; GetUVPosition(u, v, i, frequencyIndex, metaData); SetUVValue(u, v, vr, vi, 1.0); SetUVValue(-u, -v, vr, -vi, 1.0); } } break; case Flagging: if ((flags->Value(i, frequencyIndex) != 0.0L && !_invertFlagging) || (flags->Value(i, frequencyIndex) == 0.0L && _invertFlagging)) { num_t u, v; GetUVPosition(u, v, i, frequencyIndex, metaData); SetUVValue(u, v, 1, 0, 1.0); SetUVValue(-u, -v, 1, 0, 1.0); } break; } } } void UVImager::ApplyWeightsToUV() { const double normFactor = _uvWeights.Sum() / ((num_t)_uvReal.Height() * _uvReal.Width()); for (size_t y = 0; y < _uvReal.Height(); ++y) { for (size_t x = 0; x < _uvReal.Width(); ++x) { const num_t weight = _uvWeights.Value(x, y); if (weight != 0.0) { _uvReal.SetValue(x, y, _uvReal.Value(x, y) * normFactor / weight); _uvImaginary.SetValue(x, y, _uvImaginary.Value(x, y) * normFactor / weight); _uvWeights.SetValue(x, y, 1.0); } } } _uvFTReal = Image2D(); _uvFTImaginary = Image2D(); } void UVImager::SetUVValue(num_t u, num_t v, num_t r, num_t i, num_t weight) { // Nearest neighbour interpolation const long uPos = (long)std::floor(u * _uvScaling * _xRes + 0.5) + (_xRes / 2); const long vPos = (long)std::floor(v * _uvScaling * _yRes + 0.5) + (_yRes / 2); if (uPos >= 0 && uPos < (long)_xRes && vPos >= 0 && vPos < (long)_yRes) { _uvReal.AddValue(uPos, vPos, r); _uvImaginary.AddValue(uPos, vPos, i); _uvWeights.AddValue(uPos, vPos, weight); } else { if (!_ignoreBoundWarnings) { std::cout << "Warning! Baseline outside uv window (" << uPos << "," << vPos << ")." << "(subsequent out of bounds warnings will not be noted)" << std::endl; _ignoreBoundWarnings = true; } } // Linear interpolation /*long uPos = (long) floor(u*_uvScaling*_xRes+0.5L) + _xRes/2; long vPos = (long) floor(v*_uvScaling*_yRes+0.5L) + _yRes/2; if(uPos>=0 && uPos<(long) _xRes && vPos>=0 && vPos<(long) _yRes) { long double dx = (long double) uPos - (_xRes/2) - (u*_uvScaling*_xRes+0.5L); long double dy = (long double) vPos - (_yRes/2) - (v*_uvScaling*_yRes+0.5L); long double distance = sqrtn(dx*dx + dy*dy); if(distance > 1.0) distance = 1.0; weight *= distance; _uvReal.AddValue(uPos, vPos, r*weight); _uvImaginary.AddValue(uPos, vPos, i*weight); _uvWeights.AddValue(uPos, vPos, weight); } else { std::cout << "Warning! Baseline outside uv window (" << uPos << "," << vPos << ")." << std::endl; }*/ } void UVImager::SetUVFTValue(num_t u, num_t v, num_t r, num_t i, num_t weight) { for (size_t iy = 0; iy < _yResFT; ++iy) { for (size_t ix = 0; ix < _xResFT; ++ix) { const num_t x = ((num_t)ix - (_xResFT / 2)) / _uvScaling * _uvFTReal.Width(); const num_t y = ((num_t)iy - (_yResFT / 2)) / _uvScaling * _uvFTReal.Height(); // Calculate F(x,y) += f(u, v) e ^ {i 2 pi (x u + y v) } const num_t fftRotation = (u * x + v * y) * -2.0L * M_PIn; num_t fftCos = std::cos(fftRotation), fftSin = std::sin(fftRotation); _uvFTReal.AddValue(ix, iy, (fftCos * r - fftSin * i) * weight); _uvFTImaginary.AddValue(ix, iy, (fftSin * r + fftCos * i) * weight); } } } void UVImager::PerformFFT() { if (!_uvFTReal.Empty()) { _uvFTReal = Image2D::MakeZeroImage(_xRes, _yRes); _uvFTImaginary = Image2D::MakeZeroImage(_xRes, _yRes); } FFTTools::CreateFFTImage(_uvReal, _uvImaginary, _uvFTReal, _uvFTImaginary); } void UVImager::GetUVPosition(num_t& u, num_t& v, size_t timeIndex, size_t frequencyIndex, TimeFrequencyMetaDataCPtr metaData) { const num_t frequency = metaData->Band().channels[frequencyIndex].frequencyHz; u = metaData->UVW()[timeIndex].u * frequency / SpeedOfLight(); v = metaData->UVW()[timeIndex].v * frequency / SpeedOfLight(); } void UVImager::GetUVPosition(num_t& u, num_t& v, const SingleFrequencySingleBaselineData& data, const AntennaCache& cache) { const unsigned field = data.field; const num_t pointingLattitude = _fields[field].delayDirectionRA; const num_t pointingLongitude = _fields[field].delayDirectionDec; // calcTimer.Start(); const num_t earthLattitudeAngle = Date::JDToHourOfDay(Date::AipsMJDToJD(data.time)) * M_PIn / 12.0L; // long double pointingLongitude = _fields[field].delayDirectionDec; //not // used // Rotate baseline plane towards source, first rotate around z axis, then // around x axis const num_t raRotation = earthLattitudeAngle - pointingLattitude + M_PIn * 0.5L; num_t tmpCos = std::cos(raRotation); num_t tmpSin = std::sin(raRotation); const num_t dxProjected = tmpCos * cache.dx - tmpSin * cache.dy; const num_t tmpdy = tmpSin * cache.dx + tmpCos * cache.dy; tmpCos = std::cos(-pointingLongitude); tmpSin = std::sin(-pointingLongitude); const num_t dyProjected = tmpCos * tmpdy - tmpSin * cache.dz; // long double dzProjected = tmpSin*tmpdy + tmpCos*dzAnt; // we don't need it // Now, the newly projected positive z axis of the baseline points to the // field num_t baselineLength = std::sqrt(dxProjected * dxProjected + dyProjected * dyProjected); num_t baselineAngle; if (baselineLength == 0.0L) { baselineAngle = 0.0L; } else { baselineLength /= cache.wavelength; if (dxProjected > 0.0L) baselineAngle = std::atan(dyProjected / dxProjected); else baselineAngle = M_PIn - std::atan(dyProjected / -dxProjected); } u = std::cos(baselineAngle) * baselineLength; v = -std::sin(baselineAngle) * baselineLength; } num_t UVImager::GetFringeStopFrequency(size_t timeIndex, const Baseline& /*baseline*/, num_t /*delayDirectionRA*/, num_t delayDirectionDec, num_t /*frequency*/, TimeFrequencyMetaDataCPtr metaData) { // earthspeed = rad / sec const num_t earthSpeed = 2.0L * M_PIn / (24.0L * 60.0L * 60.0L); // num_t earthLattitudeAngle = // Date::JDToHourOfDay(Date::AipsMJDToJD(metaData->ObservationTimes()[timeIndex]))*M_PIn/12.0L; // num_t raSin = sinn(-delayDirectionRA - earthLattitudeAngle); // num_t raCos = cosn(-delayDirectionRA - earthLattitudeAngle); // num_t dx = baseline.antenna2.x - baseline.antenna1.x; // num_t dy = baseline.antenna2.y - baseline.antenna1.y; // num_t wavelength = 299792458.0L / frequency; return -earthSpeed * metaData->UVW()[timeIndex].u * std::cos(delayDirectionDec); } num_t UVImager::GetFringeCount(size_t timeIndexStart, size_t timeIndexEnd, unsigned channelIndex, const TimeFrequencyMetaDataCPtr metaData) { // For now, I've made this return the negative fringe count, because it does // not match with the fringe stop frequency returned above otherwise; probably // because of a mismatch in the signs of u,v,w somewhere... return -(metaData->UVW()[timeIndexEnd].w - metaData->UVW()[timeIndexStart].w) * metaData->Band().channels[channelIndex].frequencyHz / 299792458.0L; } void UVImager::InverseImage(MSMetaData& prototype, unsigned band, const Image2D& /*uvReal*/, const Image2D& /*uvImaginary*/, unsigned antenna1Index, unsigned antenna2Index) { _timeFreq = Image2D::MakeZeroImage(prototype.TimestepCount(), prototype.FrequencyCount(band)); AntennaInfo antenna1, antenna2; antenna1 = prototype.GetAntennaInfo(antenna1Index); antenna2 = prototype.GetAntennaInfo(antenna2Index); } aoflagger-v3.5.1/imagesets/0000775000175000017500000000000015146315735013752 5ustar oleoleaoflagger-v3.5.1/imagesets/filterbankset.cpp0000664000175000017500000003016215136222016017301 0ustar oleole#include "filterbankset.h" #include "../structures/date.h" #include "../util/progress/progresslistener.h" #include "../lua/telescopefile.h" #include #include namespace imagesets { FilterBankSet::FilterBankSet(const std::string& location) : location_(location), time_of_sample_(0.0), start_time_(0.0), bank1_centre_frequency(0.0), bank_channel_bandwidth_(0.0), channel_count_(0), if_count_(0), bit_count_(0), sample_count_(0), n_beams_(0), i_beam_(0), machine_id_(0), interval_count_(0), header_end_(0) { std::ifstream file(location_.c_str()); if (!file.good()) throw std::runtime_error(std::string("Error opening filterbank file ") + location_); std::string keyword = ReadString(file); if (keyword != "HEADER_START") throw std::runtime_error( "Filterbank file does not start with 'HEADER_START' -- corrupt file?"); while (file.good() && keyword != "HEADER_END") { keyword = ReadString(file); if (keyword == "tsamp") time_of_sample_ = ReadDouble(file); else if (keyword == "tstart") start_time_ = ReadDouble(file); else if (keyword == "fch1") bank1_centre_frequency = ReadDouble(file); else if (keyword == "foff") bank_channel_bandwidth_ = ReadDouble(file); else if (keyword == "nchans") channel_count_ = ReadInt(file); else if (keyword == "nifs") if_count_ = ReadInt(file); else if (keyword == "nbits") bit_count_ = ReadInt(file); else if (keyword == "nsamples") sample_count_ = ReadInt(file); else if (keyword == "machine_id") machine_id_ = ReadInt(file); else if (keyword == "telescope_id") telescope_id_ = ReadInt(file); else if (keyword == "nbeams") n_beams_ = ReadInt(file); else if (keyword == "ibeam") i_beam_ = ReadInt(file); else if (keyword == "src_raj" || keyword == "src_dej" || keyword == "az_start" || keyword == "za_start" || keyword == "refdm" || keyword == "period") ReadDouble(file); else if (keyword == "data_type" || keyword == "barycentric" || keyword == "pulsarcentric") ReadInt(file); } header_end_ = file.tellg(); if (sample_count_ == 0) { file.seekg(0, std::ios::end); const std::streampos endPos = file.tellg(); const size_t dataSize = endPos - header_end_; sample_count_ = (dataSize * 8) / channel_count_ / if_count_ / bit_count_; } Logger::Debug << "tsamp=" << time_of_sample_ << ", tstart=" << start_time_ << ", fch1=" << bank1_centre_frequency << ", foff=" << bank_channel_bandwidth_ << '\n' << "nChans=" << channel_count_ << ", nIFs=" << if_count_ << ", nBits=" << bit_count_ << ", nSamples=" << sample_count_ << "\nmachine_ID=" << machine_id_ << ", telescope_ID=" << telescope_id_ << '\n'; start_time_ = Date::MJDToAipsMJD(start_time_); const double sizeOfImage = double(channel_count_) * sample_count_ * if_count_ * sizeof(float); const double memSize = aocommon::system::TotalMemory(); interval_count_ = ceil(sizeOfImage / (memSize / 16.0)); if (interval_count_ < 1) interval_count_ = 1; if (interval_count_ * 8 > sample_count_) interval_count_ = sample_count_ / 8; Logger::Debug << round(sizeOfImage * 1e-8) * 0.1 << " GB/image required of total of " << round(memSize * 1e-8) * 0.1 << " GB of mem, splitting in " << interval_count_ << " intervals\n"; if (if_count_ != 1 && if_count_ != 4) throw std::runtime_error("Unsupported value for nIFs: " + std::to_string(if_count_)); } FilterBankSet::FilterBankSet(const FilterBankSet& source) : location_(source.location_), time_of_sample_(source.time_of_sample_), start_time_(source.start_time_), bank1_centre_frequency(source.bank1_centre_frequency), bank_channel_bandwidth_(source.bank_channel_bandwidth_), channel_count_(source.channel_count_), if_count_(source.if_count_), bit_count_(source.bit_count_), sample_count_(source.sample_count_), n_beams_(source.n_beams_), i_beam_(source.i_beam_), machine_id_(source.machine_id_), telescope_id_(source.telescope_id_), interval_count_(source.interval_count_), header_end_(source.header_end_) {} void FilterBankSet::AddReadRequest(const ImageSetIndex& index) { requests_.push_back(new BaselineData(index)); } std::unique_ptr FilterBankSet::GetNextRequested() { std::unique_ptr result = std::move(baselines_.front()); baselines_.pop(); return result; } void FilterBankSet::AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) { if (bit_count_ != 32 && bit_count_ != 8) throw std::runtime_error("This filterbank set uses " + std::to_string(bit_count_) + " bits. " "Only support for 8 or 32-bit filterbank sets has " "been added as of yet."); const size_t intervalIndex = index.Value(); const size_t startIndex = (sample_count_ * intervalIndex) / interval_count_; const size_t endIndex = (sample_count_ * (intervalIndex + 1)) / interval_count_; std::fstream file(location_.c_str()); file.seekg(header_end_ + std::streampos(startIndex * (bit_count_ / 8) * channel_count_ * if_count_)); if (bit_count_ == 32) { std::vector buffer(channel_count_ * if_count_); for (size_t x = 0; x != endIndex - startIndex; ++x) { const std::streampos pos = file.tellg(); file.read(reinterpret_cast(buffer.data()), channel_count_ * if_count_ * sizeof(float)); float* buffer_ptr = buffer.data(); for (size_t p = 0; p != if_count_; ++p) { for (size_t y = 0; y != channel_count_; ++y) { if (flags[0]->Value(x, y)) *buffer_ptr = std::numeric_limits::quiet_NaN(); ++buffer_ptr; } } file.seekp(pos); file.write(reinterpret_cast(buffer.data()), channel_count_ * if_count_ * sizeof(float)); } } else { // 8 bits std::vector buffer(channel_count_ * if_count_); for (size_t x = 0; x != endIndex - startIndex; ++x) { const std::streampos pos = file.tellg(); file.read(reinterpret_cast(buffer.data()), channel_count_ * if_count_); unsigned char* buffer_ptr = buffer.data(); for (size_t p = 0; p != if_count_; ++p) { for (size_t y = 0; y != channel_count_; ++y) { if (flags[0]->Value(x, y)) *buffer_ptr = -127; ++buffer_ptr; } } file.seekp(pos); file.write(reinterpret_cast(buffer.data()), channel_count_ * if_count_); } } } std::string FilterBankSet::TelescopeName() { return TelescopeFile::TelescopeName(TelescopeFile::GENERIC_TELESCOPE); } void FilterBankSet::Initialize() {} void FilterBankSet::PerformReadRequests(ProgressListener& progress) { if (bit_count_ != 32 && bit_count_ != 8) throw std::runtime_error("This filterbank set uses " + std::to_string(bit_count_) + " bits. " "Only support for 8 or 32-bit filterbank sets has " "been added as of yet."); const size_t n_requests = requests_.size(); while (!requests_.empty()) { std::unique_ptr baseline(std::move(requests_.front())); requests_.pop_front(); const size_t intervalIndex = baseline->Index().Value(); const size_t startIndex = (sample_count_ * intervalIndex) / interval_count_; const size_t endIndex = startIndex + (sample_count_ * (intervalIndex + 1)) / interval_count_; std::ifstream file(location_.c_str()); file.seekg(header_end_ + std::streampos(startIndex * (bit_count_ / 8) * channel_count_ * if_count_)); std::vector images(if_count_); const size_t width = endIndex - startIndex; for (Image2DPtr& image : images) image = Image2D::CreateUnsetImagePtr(width, channel_count_); std::vector masks(if_count_); for (Mask2DPtr& mask : masks) mask = Mask2D::CreateUnsetMaskPtr(width, channel_count_); if (bit_count_ == 32) { std::vector buffer(channel_count_ * if_count_); for (size_t x = 0; x != width; ++x) { progress.OnProgress(x + width * (n_requests - requests_.size() - 1), width * n_requests); file.read(reinterpret_cast(buffer.data()), channel_count_ * if_count_ * sizeof(float)); const float* buffer_ptr = buffer.data(); for (size_t p = 0; p != if_count_; ++p) { for (size_t y = 0; y != channel_count_; ++y) { images[p]->SetValue(x, y, *buffer_ptr); masks[p]->SetValue(x, y, !std::isfinite(*buffer_ptr)); ++buffer_ptr; } } } } else { std::vector buffer(channel_count_ * if_count_); for (size_t x = 0; x != width; ++x) { progress.OnProgress(x + width * (n_requests - requests_.size() - 1), width * n_requests); file.read(reinterpret_cast(buffer.data()), channel_count_ * if_count_); const unsigned char* buffer_ptr = buffer.data(); for (size_t p = 0; p != if_count_; ++p) { for (size_t y = 0; y != channel_count_; ++y) { images[p]->SetValue(x, y, static_cast(*buffer_ptr)); masks[p]->SetValue(x, y, *buffer_ptr == 255); ++buffer_ptr; } } } } TimeFrequencyData tfData; if (if_count_ == 1) { tfData = TimeFrequencyData(TimeFrequencyData::AmplitudePart, aocommon::Polarization::StokesI, images[0]); tfData.SetGlobalMask(masks[0]); } else { tfData = TimeFrequencyData::FromStokes(images[0], images[1], images[2], images[3]); for (size_t i = 0; i != if_count_; ++i) tfData.SetIndividualPolarizationMasks(masks.data()); } const TimeFrequencyMetaDataPtr metaData(new TimeFrequencyMetaData()); AntennaInfo antenna; antenna.diameter = 0; antenna.id = 0; antenna.mount = "unknown"; antenna.name = "unknown"; antenna.position = EarthPosition(); antenna.station = "unknown"; metaData->SetAntenna1(antenna); metaData->SetAntenna2(antenna); BandInfo band; band.windowIndex = 0; for (size_t ch = 0; ch != channel_count_; ++ch) { ChannelInfo channel; channel.frequencyHz = 1e6 * (bank1_centre_frequency + bank_channel_bandwidth_ * ch); channel.effectiveBandWidthHz = 1e6 * std::fabs(bank_channel_bandwidth_); channel.frequencyIndex = ch; channel.channelWidthHz = 1e6 * std::fabs(bank_channel_bandwidth_); channel.resolutionHz = 1e6 * std::fabs(bank_channel_bandwidth_); band.channels.push_back(channel); } metaData->SetBand(band); std::vector observationTimes(endIndex - startIndex); for (size_t t = startIndex; t != endIndex; ++t) observationTimes[t - startIndex] = (start_time_ + time_of_sample_ * t); metaData->SetObservationTimes(observationTimes); metaData->SetValueDescription("Power"); baseline->SetData(tfData); baseline->SetMetaData(metaData); baselines_.emplace(std::move(baseline)); } progress.OnFinish(); } void FilterBankSet::PerformWriteDataTask( const ImageSetIndex& index, std::vector realImages, std::vector imaginaryImages) { throw std::runtime_error( "Can't write data back to filter bank format: not implemented"); } std::string FilterBankSet::Description(const ImageSetIndex& index) const { std::ostringstream str; str << "Filterbank set -- interval " << (index.Value() + 1) << '/' << interval_count_; return str.str(); } } // namespace imagesets aoflagger-v3.5.1/imagesets/joinedspwset.h0000664000175000017500000002624714752462134016652 0ustar oleole #ifndef JOINED_SPW_SET_H #define JOINED_SPW_SET_H #include "imageset.h" #include "msimageset.h" #include "../structures/msmetadata.h" #include #include #include #include #include #include namespace imagesets { using Sequence = MSMetaData::Sequence; class JoinedSPWSet final : public IndexableSet { public: /** * @param msImageSet An already initialized image set of which ownership is * transferred to this class. */ explicit JoinedSPWSet(std::unique_ptr msImageSet) : _msImageSet(std::move(msImageSet)) { const std::vector& sequences = _msImageSet->Sequences(); size_t nBands = _msImageSet->BandCount(); _nChannels.resize(nBands); for (size_t b = 0; b != nBands; ++b) _nChannels[b] = _msImageSet->GetBandInfo(b).channels.size(); std::map>> jsMap; for (size_t sequenceIndex = 0; sequenceIndex != sequences.size(); ++sequenceIndex) { const MSMetaData::Sequence& s = sequences[sequenceIndex]; Sequence js(s.antenna1, s.antenna2, 0, s.sequenceId, s.fieldId); // TODO Central frequency instead of spw id is a better idea jsMap[js].emplace_back(s.spw, sequenceIndex); } for (auto& js : jsMap) { std::sort(js.second.begin(), js.second.end()); _joinedSequences.emplace_back(js.first, std::move(js.second)); } } JoinedSPWSet(const JoinedSPWSet& source) = delete; JoinedSPWSet& operator=(const JoinedSPWSet& source) = delete; ~JoinedSPWSet() override{}; std::unique_ptr Clone() override { std::unique_ptr newSet(new JoinedSPWSet()); newSet->_msImageSet.reset(new MSImageSet(*_msImageSet)); newSet->_joinedSequences = _joinedSequences; newSet->_nChannels = _nChannels; return newSet; } size_t Size() const override { return _joinedSequences.size(); } std::string Description(const ImageSetIndex& index) const override; void Initialize() override {} std::string Name() const override { return _msImageSet->Name() + " (SPWs joined)"; } std::vector Files() const override { return _msImageSet->Files(); } void AddReadRequest(const ImageSetIndex& index) override { const std::vector>& indexInformation = _joinedSequences[index.Value()].bandAndOriginalSeq; for (const std::pair& spwAndSeq : indexInformation) { ImageSetIndex msIndex(_msImageSet->Size(), spwAndSeq.second); _msImageSet->AddReadRequest(msIndex); } _requests.push_back(index.Value()); } /** * Combines the baseline data of multiple bands in one band. * * @param data The baseline data of the individual bands. * @param totalHeight The sum of the channels in all bands in @a data. * @param index The index of the final baseline. */ static std::unique_ptr CombineBaselineData( std::vector>&& data, size_t totalHeight, const ImageSetIndex& index) { // Combine the images TimeFrequencyData tfData(data[0]->Data()); size_t width = tfData.ImageWidth(); for (size_t imgIndex = 0; imgIndex != tfData.ImageCount(); ++imgIndex) { size_t chIndex = 0; Image2DPtr img = Image2D::CreateUnsetImagePtr(width, totalHeight); for (size_t i = 0; i != data.size(); ++i) { Image2DCPtr src = data[i]->Data().GetImage(imgIndex); for (size_t y = 0; y != src->Height(); ++y) { num_t* destPtr = img->ValuePtr(0, y + chIndex); const num_t* srcPtr = src->ValuePtr(0, y); for (size_t x = 0; x != src->Width(); ++x) destPtr[x] = srcPtr[x]; } chIndex += data[i]->Data().ImageHeight(); } tfData.SetImage(imgIndex, img); } // Combine the masks for (size_t maskIndex = 0; maskIndex != tfData.MaskCount(); ++maskIndex) { size_t chIndex = 0; Mask2DPtr mask = Mask2D::CreateUnsetMaskPtr(width, totalHeight); for (size_t i = 0; i != data.size(); ++i) { Mask2DCPtr src = data[i]->Data().GetMask(maskIndex); for (size_t y = 0; y != src->Height(); ++y) { bool* destPtr = mask->ValuePtr(0, y + chIndex); const bool* srcPtr = src->ValuePtr(0, y); for (size_t x = 0; x != src->Width(); ++x) destPtr[x] = srcPtr[x]; } chIndex += data[i]->Data().ImageHeight(); } tfData.SetMask(maskIndex, mask); } // Combine the metadata TimeFrequencyMetaDataPtr metaData( new TimeFrequencyMetaData(*data[0]->MetaData())); BandInfo band = metaData->Band(); size_t chIndex = band.channels.size(); band.channels.resize(totalHeight); for (size_t i = 1; i != data.size(); ++i) { const BandInfo& srcBand = data[i]->MetaData()->Band(); for (size_t ch = 0; ch != srcBand.channels.size(); ++ch) band.channels[ch + chIndex] = srcBand.channels[ch]; chIndex += srcBand.channels.size(); } metaData->SetBand(band); return std::make_unique(std::move(tfData), std::move(metaData), index); } void PerformReadRequests(class ProgressListener& progress) override { _msImageSet->PerformReadRequests(progress); for (size_t request : _requests) { const std::vector>& indexInformation = _joinedSequences[request].bandAndOriginalSeq; std::vector> data; size_t totalHeight = 0; for (size_t i = 0; i != indexInformation.size(); ++i) { data.emplace_back(_msImageSet->GetNextRequested()); totalHeight += data.back()->Data().ImageHeight(); } const Sequence seq = _joinedSequences[request].sequence; std::optional index = Index(seq.antenna1, seq.antenna2, seq.spw, seq.sequenceId); if (!index) throw std::runtime_error("Baseline not found"); _baselineData.push_back( CombineBaselineData(std::move(data), totalHeight, *index)); } _requests.clear(); } std::unique_ptr GetNextRequested() override { std::unique_ptr result = std::move(_baselineData.front()); _baselineData.pop_front(); return result; } void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) override { const std::vector>& indexInformation = _joinedSequences[index.Value()].bandAndOriginalSeq; size_t width = flags.front()->Width(); size_t chIndex = 0; for (size_t spw = 0; spw != indexInformation.size(); ++spw) { const std::pair& spwAndSeq = indexInformation[spw]; std::vector spwFlags(flags.size()); for (size_t m = 0; m != flags.size(); ++m) { Mask2DPtr spwMask = Mask2D::CreateUnsetMaskPtr(width, _nChannels[spwAndSeq.first]); for (size_t y = 0; y != _nChannels[spwAndSeq.first]; ++y) { const bool* srcPtr = flags[m]->ValuePtr(0, y + chIndex); bool* destPtr = spwMask->ValuePtr(0, y); for (size_t x = 0; x != flags[m]->Width(); ++x) destPtr[x] = srcPtr[x]; } spwFlags[m] = std::move(spwMask); } chIndex += _nChannels[spw]; ImageSetIndex msIndex(Size(), spwAndSeq.second); _msImageSet->AddWriteFlagsTask(msIndex, spwFlags); } } void PerformWriteFlagsTask() override { _msImageSet->PerformWriteFlagsTask(); } BaselineReaderPtr Reader() const override { return _msImageSet->Reader(); } MSImageSet& msImageSet() { return *_msImageSet; } size_t AntennaCount() const override { return _msImageSet->AntennaCount(); } size_t GetAntenna1(const ImageSetIndex& index) const override { return _joinedSequences[index.Value()].sequence.antenna1; } size_t GetAntenna2(const ImageSetIndex& index) const override { return _joinedSequences[index.Value()].sequence.antenna2; } size_t GetBand(const ImageSetIndex& index) const override { return 0; } size_t GetField(const ImageSetIndex& index) const override { return _joinedSequences[index.Value()].sequence.fieldId; } size_t GetSequenceId(const ImageSetIndex& index) const override { return _joinedSequences[index.Value()].sequence.sequenceId; } AntennaInfo GetAntennaInfo(unsigned antennaIndex) const override { return _msImageSet->GetAntennaInfo(antennaIndex); } size_t BandCount() const override { return 1; } BandInfo GetBandInfo(unsigned bandIndex) const override { BandInfo band = _msImageSet->GetBandInfo(0); size_t chIndex = band.channels.size(); for (size_t i = 1; i != _msImageSet->BandCount(); ++i) { const BandInfo srcBand = _msImageSet->GetBandInfo(i); band.channels.resize(chIndex + srcBand.channels.size()); for (size_t ch = 0; ch != srcBand.channels.size(); ++ch) band.channels[ch + chIndex] = srcBand.channels[ch]; chIndex += srcBand.channels.size(); } return band; } size_t SequenceCount() const override { return _msImageSet->SequenceCount(); } std::optional Index(size_t antenna1, size_t antenna2, size_t bandIndex, size_t sequenceId) const override { for (size_t i = 0; i != _joinedSequences.size(); ++i) { const Sequence seq = _joinedSequences[i].sequence; bool antennaMatch = (seq.antenna1 == antenna1 && seq.antenna2 == antenna2) || (seq.antenna1 == antenna2 && seq.antenna2 == antenna1); if (antennaMatch && seq.sequenceId == sequenceId) { return ImageSetIndex(Size(), i); } } std::stringstream str; str << "Baseline not found: " << "antenna1=" << antenna1 << ", " << "antenna2=" << antenna2 << ", " << "sequenceId=" << sequenceId; throw std::runtime_error(str.str()); } FieldInfo GetFieldInfo(unsigned fieldIndex) const override { return _msImageSet->GetFieldInfo(fieldIndex); } private: JoinedSPWSet() = default; std::unique_ptr _msImageSet; struct JoinedSequence { JoinedSequence(const Sequence& s, std::vector>&& bandSeq) : sequence(s), bandAndOriginalSeq(std::move(bandSeq)) {} Sequence sequence; std::vector> bandAndOriginalSeq; }; std::vector _joinedSequences; std::vector _requests; std::list> _baselineData; std::vector _nChannels; }; std::string JoinedSPWSet::Description(const ImageSetIndex& index) const { const Sequence& sequence = _joinedSequences[index.Value()].sequence; size_t antenna1 = sequence.antenna1, antenna2 = sequence.antenna2, sequenceId = sequence.sequenceId; AntennaInfo info1 = _msImageSet->GetAntennaInfo(antenna1); AntennaInfo info2 = _msImageSet->GetAntennaInfo(antenna2); std::stringstream sstream; sstream << info1.station << ' ' << info1.name << " x " << info2.station << ' ' << info2.name << " (joined spws)"; if (_msImageSet->SequenceCount() > 1) { sstream << ", seq " << sequenceId; } return sstream.str(); } } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/pngreader.h0000664000175000017500000000176414752462134016100 0ustar oleole#ifndef PNGREADER_H #define PNGREADER_H #include #include #include #include #include #include "../structures/types.h" #include "../lua/telescopefile.h" #include "singleimageset.h" #include "../util/logger.h" namespace imagesets { class PngReader final : public SingleImageSet { public: explicit PngReader(const std::string& path) : SingleImageSet(), _path(path) {} std::unique_ptr Clone() override { return nullptr; } void Initialize() override {} std::string Name() const override { return "Png file"; } std::vector Files() const override { return std::vector{_path}; } std::string BaselineDescription() override { return Name(); } std::string TelescopeName() override { return TelescopeFile::TelescopeName(TelescopeFile::GENERIC_TELESCOPE); } std::unique_ptr Read(class ProgressListener& progress) override; private: std::string _path; }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/h5imageset.h0000664000175000017500000000634414752462134016163 0ustar oleole#ifndef IMAGESETS_H5_IMAGE_SET_H_ #define IMAGESETS_H5_IMAGE_SET_H_ #include #include #include #include #include #include #include #include "imageset.h" class ProgressListener; namespace imagesets { /** * This class provides the support for the LOFAR beam-formed file format. * There's a tutorial on this format made by Cees Bassa: * * - https://github.com/cbassa/lofar_bf_tutorials/tree/master/solutions * * He summarized the most important bits (to read the axes & data) as (in Python * code): * @code{.py} * stokes = h5["/SUB_ARRAY_POINTING_000/BEAM_000/STOKES_0"] * freq = * h5["/SUB_ARRAY_POINTING_000/BEAM_000/COORDINATES/COORDINATE_1"].attrs["AXIS_VALUES_WORLD"] * * 1e-6 tsamp = * h5["/SUB_ARRAY_POINTING_000/BEAM_000/COORDINATES/COORDINATE_0"].attrs["INCREMENT"] * t = tsamp * np.arange(stokes.shape[0]) * @endcode * * This is more or less what this class does. The 'stokes' data needs to be * transposed from frequency first to time first. The attribute 'TELESCOPE' is * read for determining the telescope, but as far as we know only LOFAR uses * this format. */ class H5ImageSet final : public ImageSet { public: explicit H5ImageSet(const std::string& path); H5ImageSet(const H5ImageSet& source) : path_(source.path_), telescope_name_(source.telescope_name_), interval_start_(source.interval_start_), interval_end_(source.interval_end_), requests_(), // Ongoing requests should not be copied baseline_buffer_() // Already read data should not be copied {} H5ImageSet& operator=(const H5ImageSet& rhs) = delete; size_t Size() const override { return 1; } std::string Description(const ImageSetIndex& index) const override; std::unique_ptr Clone() override { return std::make_unique(*this); } void Initialize() override; std::string Name() const override { return "H5/raw"; } std::vector Files() const override { return std::vector{path_}; } std::string TelescopeName() override { return telescope_name_; } void SetInterval(std::optional start, std::optional end) { interval_start_ = start; interval_end_ = end; } bool HasCrossCorrelations() const override { return false; } void AddReadRequest(const ImageSetIndex& index) override { requests_.emplace_back(index); } void PerformReadRequests(ProgressListener& progress) override; std::unique_ptr GetNextRequested() override { std::unique_ptr baseline(std::move(baseline_buffer_.front())); baseline_buffer_.pop(); return baseline; } void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) override; void PerformWriteFlagsTask() override {} private: std::string path_; std::string telescope_name_; std::optional interval_start_; std::optional interval_end_; std::vector requests_; std::queue> baseline_buffer_; std::unique_ptr LoadData(ProgressListener& progress, const ImageSetIndex& index); }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/msimageset.cpp0000664000175000017500000001642314752462134016620 0ustar oleole#include #include #include #include #include "../algorithms/timefrequencystatistics.h" #include "msimageset.h" #include "../msio/directbaselinereader.h" #include "../msio/reorderingbaselinereader.h" #include "../msio/memorybaselinereader.h" #include "../util/logger.h" namespace imagesets { void MSImageSet::Initialize() { Logger::Debug << "Initializing image set...\n"; Logger::Debug << "Antennas: " << _metaData.AntennaCount() << '\n'; _sequences = _metaData.GetSequences(); Logger::Debug << "Unique sequences: " << _sequences.size() << '\n'; if (_sequences.empty()) throw std::runtime_error( "Trying to open a measurement set with no sequences"); initReader(); _bandCount = _metaData.BandCount(); _fieldCount = _metaData.FieldCount(); _sequencesPerBaselineCount = _metaData.SequenceCount(); Logger::Debug << "Bands: " << _bandCount << '\n'; } void MSImageSet::initReader() { if (_reader == nullptr) { BaselineIOMode selected_mode = _ioMode; if (selected_mode == AutoReadMode) { if (MemoryBaselineReader::IsEnoughMemoryAvailable( BaselineReader::MeasurementSetIntervalDataSize( _msFile, _intervalStart, _intervalEnd))) selected_mode = MemoryReadMode; else selected_mode = ReorderingReadMode; } switch (selected_mode) { case ReorderingReadMode: { ReorderingBaselineReader* indirectReader = new ReorderingBaselineReader(_msFile); indirectReader->SetReadUVW(_readUVW); _reader = BaselineReaderPtr(indirectReader); } break; case DirectReadMode: _reader = BaselineReaderPtr(new DirectBaselineReader(_msFile)); break; case MemoryReadMode: _reader = BaselineReaderPtr(new MemoryBaselineReader(_msFile)); break; case AutoReadMode: assert(false); break; } } _reader->SetDataColumnName(_dataColumnName); _reader->SetInterval(_intervalStart, _intervalEnd); _reader->SetReadFlags(_readFlags); _reader->SetReadData(true); } size_t MSImageSet::EndTimeIndex(const ImageSetIndex& index) const { return _reader->MetaData() .GetObservationTimesSet(GetSequenceId(index)) .size(); } std::vector MSImageSet::ObservationTimesVector( const ImageSetIndex& index) { // StartIndex(msIndex), EndIndex(msIndex) const unsigned sequenceId = _sequences[index.Value()].sequenceId; const std::set& obsTimesSet = _reader->MetaData().GetObservationTimesSet(sequenceId); std::vector obs(obsTimesSet.begin(), obsTimesSet.end()); return obs; } TimeFrequencyMetaDataCPtr MSImageSet::createMetaData(const ImageSetIndex& index, std::vector& uvw) { TimeFrequencyMetaData* metaData = new TimeFrequencyMetaData(); metaData->SetAntenna1(_metaData.GetAntennaInfo(GetAntenna1(index))); metaData->SetAntenna2(_metaData.GetAntennaInfo(GetAntenna2(index))); metaData->SetBand(_metaData.GetBandInfo(GetBand(index))); metaData->SetField(_metaData.GetFieldInfo(GetField(index))); metaData->SetObservationTimes(ObservationTimesVector(index)); if (_reader != nullptr) { metaData->SetUVW(uvw); } return TimeFrequencyMetaDataCPtr(metaData); } std::string MSImageSet::Description(const ImageSetIndex& index) const { std::stringstream sstream; const MSMetaData::Sequence& sequence = _sequences[index.Value()]; size_t antenna1 = sequence.antenna1, antenna2 = sequence.antenna2, band = sequence.spw, sequenceId = sequence.sequenceId; const AntennaInfo info1 = GetAntennaInfo(antenna1); const AntennaInfo info2 = GetAntennaInfo(antenna2); sstream << info1.station << ' ' << info1.name << " x " << info2.station << ' ' << info2.name; if (BandCount() > 1) { BandInfo bandInfo = GetBandInfo(band); const double bandStart = round(bandInfo.channels.front().frequencyHz / 100000.0) / 10.0; const double bandEnd = round(bandInfo.channels.back().frequencyHz / 100000.0) / 10.0; sstream << ", spw " << band << " (" << bandStart << "MHz -" << bandEnd << "MHz)"; } if (SequenceCount() > 1) { sstream << ", seq " << sequenceId; } return sstream.str(); } size_t MSImageSet::findBaselineIndex(size_t antenna1, size_t antenna2, size_t band, size_t sequenceId) const { size_t index = 0; for (std::vector::const_iterator i = _sequences.begin(); i != _sequences.end(); ++i) { const bool antennaMatch = (i->antenna1 == antenna1 && i->antenna2 == antenna2) || (i->antenna1 == antenna2 && i->antenna2 == antenna1); if (antennaMatch && i->spw == band && i->sequenceId == sequenceId) { return index; } ++index; } return not_found; } void MSImageSet::AddReadRequest(const ImageSetIndex& index) { const BaselineData newRequest(index); _baselineData.push_back(newRequest); } void MSImageSet::PerformReadRequests(class ProgressListener& progress) { for (BaselineData& bd : _baselineData) { _reader->AddReadRequest(GetAntenna1(bd.Index()), GetAntenna2(bd.Index()), GetBand(bd.Index()), GetSequenceId(bd.Index()), StartTimeIndex(bd.Index()), EndTimeIndex(bd.Index())); } _reader->PerformReadRequests(progress); for (std::vector::iterator i = _baselineData.begin(); i != _baselineData.end(); ++i) { if (!i->Data().IsEmpty()) throw std::runtime_error( "ReadRequest() called, but a previous read request was not " "completely processed by calling GetNextRequested()."); std::vector uvw; const TimeFrequencyData data = _reader->GetNextResult(uvw); i->SetData(data); const TimeFrequencyMetaDataCPtr metaData = createMetaData(i->Index(), uvw); i->SetMetaData(metaData); } } std::unique_ptr MSImageSet::GetNextRequested() { std::unique_ptr top(new BaselineData(_baselineData.front())); _baselineData.erase(_baselineData.begin()); if (top->Data().IsEmpty()) throw std::runtime_error( "Calling GetNextRequested(), but requests were not read with " "LoadRequests."); return top; } void MSImageSet::AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) { const size_t seqIndex = index.Value(); initReader(); const size_t a1 = _sequences[seqIndex].antenna1; const size_t a2 = _sequences[seqIndex].antenna2; const size_t b = _sequences[seqIndex].spw; const size_t s = _sequences[seqIndex].sequenceId; std::vector allFlags; if (flags.size() > _reader->Polarizations().size()) { throw std::runtime_error( "Trying to write more polarizations to image set than available"); } else if (flags.size() < _reader->Polarizations().size()) { if (flags.size() == 1) for (size_t i = 0; i < _reader->Polarizations().size(); ++i) allFlags.emplace_back(flags[0]); else throw std::runtime_error( "Incorrect number of polarizations in write action"); } else { allFlags = flags; } _reader->AddWriteTask(allFlags, a1, a2, b, s); } void MSImageSet::PerformWriteFlagsTask() { _reader->PerformFlagWriteRequests(); } } // namespace imagesets aoflagger-v3.5.1/imagesets/msimageset.h0000664000175000017500000001325715065216451016264 0ustar oleole#ifndef MSIMAGESET_H #define MSIMAGESET_H #include #include #include #include #include #include #include "../structures/antennainfo.h" #include "../structures/timefrequencydata.h" #include "../structures/timefrequencymetadata.h" #include "../structures/msmetadata.h" #include "../msio/baselinereader.h" #include "indexableset.h" #include "../util/logger.h" namespace imagesets { class MSImageSet final : public IndexableSet { public: MSImageSet(const std::string& location, BaselineIOMode ioMode) : _msFile(location), _metaData(location), _reader(), _dataColumnName("DATA"), _intervalStart(), _intervalEnd(), _bandCount(0), _fieldCount(0), _sequencesPerBaselineCount(0), _readFlags(true), _readUVW(false), _ioMode(ioMode) {} MSImageSet(const MSImageSet&) = default; ~MSImageSet() = default; size_t StartTimeIndex(const ImageSetIndex& index) const { return 0; } size_t EndTimeIndex(const ImageSetIndex& index) const; std::unique_ptr Clone() final { return std::make_unique(*this); } size_t Size() const final { return _sequences.size(); } std::string Description(const ImageSetIndex& index) const final; std::string Name() const final { return _metaData.Path(); } std::vector Files() const final { return std::vector{_metaData.Path()}; } void AddReadRequest(const ImageSetIndex& index) final; void PerformReadRequests(class ProgressListener& progress) final; std::unique_ptr GetNextRequested() final; void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) final; void PerformWriteFlagsTask() final; void Initialize() final; void PerformWriteDataTask(const ImageSetIndex& index, std::vector realImages, std::vector imaginaryImages) final { _reader->PerformDataWriteTask(realImages, imaginaryImages, GetAntenna1(index), GetAntenna2(index), GetBand(index), GetSequenceId(index)); } BaselineReaderPtr Reader() const final { return _reader; } MSMetaData& MetaData() { return _metaData; } size_t GetAntenna1(const ImageSetIndex& index) const final { return _sequences[index.Value()].antenna1; } size_t GetAntenna2(const ImageSetIndex& index) const final { return _sequences[index.Value()].antenna2; } size_t GetBand(const ImageSetIndex& index) const final { return _sequences[index.Value()].spw; } size_t GetField(const ImageSetIndex& index) const final { return _sequences[index.Value()].fieldId; } size_t GetSequenceId(const ImageSetIndex& index) const final { return _sequences[index.Value()].sequenceId; } std::optional Index(size_t antenna1, size_t antenna2, size_t bandIndex, size_t sequenceId) const final { size_t value = findBaselineIndex(antenna1, antenna2, bandIndex, sequenceId); if (value == not_found) return std::optional(); else return ImageSetIndex(Size(), value); } const std::string& DataColumnName() const { return _dataColumnName; } void SetDataColumnName(const std::string& name) { if (_reader != nullptr) throw std::runtime_error( "Trying to set data column after creating the reader!"); _dataColumnName = name; } size_t BandCount() const final { return _bandCount; } class ::AntennaInfo GetAntennaInfo(unsigned antennaIndex) const final { return _metaData.GetAntennaInfo(antennaIndex); } class ::BandInfo GetBandInfo(unsigned bandIndex) const final { return _metaData.GetBandInfo(bandIndex); } size_t SequenceCount() const final { return _sequencesPerBaselineCount; } size_t AntennaCount() const final { return _metaData.AntennaCount(); } class ::FieldInfo GetFieldInfo(unsigned fieldIndex) const final { return _metaData.GetFieldInfo(fieldIndex); } std::vector ObservationTimesVector(const ImageSetIndex& index); size_t FieldCount() const { return _fieldCount; } void SetReadFlags(bool readFlags) { _readFlags = readFlags; } void SetReadUVW(bool readUVW) { _readUVW = readUVW; } const std::vector& Sequences() const { return _sequences; } void SetInterval(std::optional start, std::optional end) { _intervalStart = start; _intervalEnd = end; if (start) _metaData.SetIntervalStart(*start); if (end) _metaData.SetIntervalEnd(*end); } private: MSImageSet(const std::string& location, BaselineReaderPtr reader) : _msFile(location), _metaData(location), _reader(reader), _dataColumnName("DATA"), _readFlags(true), _readUVW(false), _ioMode(AutoReadMode) {} void initReader(); static const size_t not_found = std::numeric_limits::max(); size_t findBaselineIndex(size_t antenna1, size_t antenna2, size_t band, size_t sequenceId) const; TimeFrequencyMetaDataCPtr createMetaData(const ImageSetIndex& index, std::vector& uvw); const std::string _msFile; MSMetaData _metaData; BaselineReaderPtr _reader; std::string _dataColumnName; std::optional _intervalStart, _intervalEnd; std::vector _sequences; size_t _bandCount, _fieldCount, _sequencesPerBaselineCount; bool _readFlags, _readUVW; BaselineIOMode _ioMode; std::vector _baselineData; }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/coaddedimageset.h0000664000175000017500000001367014752462134017232 0ustar oleole#ifndef COADDED_IMAGE_SET_H #define COADDED_IMAGE_SET_H #include "imageset.h" #include "msimageset.h" #include "../structures/msmetadata.h" #include "../util/logger.h" #include #include #include #include #include #include namespace imagesets { using Sequence = MSMetaData::Sequence; class CoaddedImageSet final : public IndexableSet { public: explicit CoaddedImageSet(const std::vector& filenames, BaselineIOMode ioMode) { if (filenames.empty()) throw std::runtime_error( "Coadding of image sets was requested, but list of measurement sets " "to coadd is empty"); if (filenames.size() == 1) throw std::runtime_error( "Coadding of image sets was requested, but only one measurement set " "is given"); for (const std::string& path : filenames) { std::unique_ptr imageSet(new MSImageSet(path, ioMode)); _msImageSets.emplace_back(std::move(imageSet)); } } CoaddedImageSet(const CoaddedImageSet& source) = delete; CoaddedImageSet& operator=(const CoaddedImageSet& source) = delete; std::unique_ptr Clone() override { std::unique_ptr newSet(new CoaddedImageSet()); for (const std::unique_ptr& imageSet : _msImageSets) { newSet->_msImageSets.emplace_back( std::unique_ptr(new MSImageSet(*imageSet))); } return newSet; } size_t Size() const override { return _msImageSets[0]->Size(); } std::string Description(const ImageSetIndex& index) const override; void Initialize() override { for (std::unique_ptr& imageSet : _msImageSets) imageSet->Initialize(); } std::string Name() const override { return "Coadded set (" + _msImageSets.front()->Name() + " ...)"; } std::vector Files() const override { return _msImageSets.front()->Files(); } void AddReadRequest(const ImageSetIndex& index) override { for (size_t i = 0; i != _msImageSets.size(); ++i) { _msImageSets[i]->AddReadRequest(index); } } void PerformReadRequests(class ProgressListener& progress) override { for (size_t i = 0; i != _msImageSets.size(); ++i) { _msImageSets[i]->PerformReadRequests(progress); // TODO } } std::unique_ptr GetNextRequested() override { std::unique_ptr data = _msImageSets.front()->GetNextRequested(); TimeFrequencyData tfData(data->Data()); std::vector images; for (size_t i = 0; i != tfData.PolarizationCount(); ++i) { TimeFrequencyData polAmplitude = tfData.MakeFromPolarizationIndex(i).Make( TimeFrequencyData::AmplitudePart); images.emplace_back(Image2DPtr(new Image2D(*polAmplitude.GetImage(0)))); } for (size_t i = 1; i != _msImageSets.size(); ++i) { std::unique_ptr addedData = _msImageSets[i]->GetNextRequested(); if (addedData->Data().PolarizationCount() != images.size()) throw std::runtime_error( "Coadded images have different number of polarizations"); for (size_t j = 0; j != images.size(); ++j) { TimeFrequencyData polAmplitude = addedData->Data().MakeFromPolarizationIndex(j).Make( TimeFrequencyData::AmplitudePart); images[j]->operator+=(*polAmplitude.GetImage(0)); } } Image2DCPtr zeroImage = Image2D::CreateSetImagePtr( images.front()->Width(), images.front()->Height(), 0.0); for (size_t i = 0; i != tfData.PolarizationCount(); ++i) { TimeFrequencyData pol = tfData.MakeFromPolarizationIndex(i); pol.SetImage(0, images[i]); pol.SetImage(1, zeroImage); tfData.SetPolarizationData(i, std::move(pol)); } data->SetData(tfData); return data; } void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) override {} void PerformWriteFlagsTask() override {} BaselineReaderPtr Reader() const override { return _msImageSets.front()->Reader(); } size_t AntennaCount() const override { return _msImageSets.front()->AntennaCount(); } size_t GetAntenna1(const ImageSetIndex& index) const override { return _msImageSets.front()->GetAntenna1(index); } size_t GetAntenna2(const ImageSetIndex& index) const override { return _msImageSets.front()->GetAntenna2(index); } size_t GetBand(const ImageSetIndex& index) const override { return _msImageSets.front()->GetBand(index); } size_t GetField(const ImageSetIndex& index) const override { return _msImageSets.front()->GetField(index); } size_t GetSequenceId(const ImageSetIndex& index) const override { return _msImageSets.front()->GetSequenceId(index); } AntennaInfo GetAntennaInfo(unsigned antennaIndex) const override { return _msImageSets.front()->GetAntennaInfo(antennaIndex); } size_t BandCount() const override { return _msImageSets.front()->BandCount(); } BandInfo GetBandInfo(unsigned bandIndex) const override { return _msImageSets.front()->GetBandInfo(bandIndex); } size_t SequenceCount() const override { return _msImageSets.front()->SequenceCount(); } std::optional Index(size_t antenna1, size_t antenna2, size_t bandIndex, size_t sequenceId) const override { return _msImageSets[0]->Index(antenna1, antenna2, bandIndex, sequenceId); } FieldInfo GetFieldInfo(unsigned fieldIndex) const override { return _msImageSets.front()->GetFieldInfo(fieldIndex); } const std::vector>& MSImageSets() { return _msImageSets; } private: CoaddedImageSet() {} std::vector> _msImageSets; }; std::string CoaddedImageSet::Description(const ImageSetIndex& index) const { return _msImageSets[0]->Description(index) + " (coadded)"; } } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/sdhdfimageset.cpp0000664000175000017500000003070014752462134017263 0ustar oleole#include "sdhdfimageset.h" #include #include "../util/logger.h" #include "../util/progress/subtasklistener.h" #include "aocommon/uvector.h" namespace imagesets { struct ObsParams { double mjd; }; SdhdfImageSet::SdhdfImageSet(const std::string& path) : _path(path) { const H5::H5File file(_path, H5F_ACC_RDONLY); const H5::DataSet headerSet = file.openDataSet("metadata/primary_header"); const H5::DataSpace headerSpace = headerSet.getSpace(); if (headerSpace.getSimpleExtentNdims() != 1) throw std::runtime_error( "Incorrect dimensionality of metadata/primary_header"); hsize_t dims; headerSpace.getSimpleExtentDims(&dims); if (dims != 1) throw std::runtime_error( "Incorrect nr of elements in metadata/primary_header data set"); const H5::CompType headerType(sizeof(Header)); headerType.insertMember("TELESCOPE", HOFFSET(Header, telescopeName), H5::StrType(H5::PredType::C_S1, 64)); headerType.insertMember("N_BEAMS", HOFFSET(Header, nBeams), H5::PredType::NATIVE_INT); Header header; headerSet.read(&header, headerType); _beams.resize(header.nBeams); const H5::CompType bandParamsType(sizeof(BandParams)); bandParamsType.insertMember("LABEL", HOFFSET(BandParams, label), H5::StrType(H5::PredType::C_S1, 64)); bandParamsType.insertMember("CENTRE_FREQ", HOFFSET(BandParams, centreFrequency), H5::PredType::NATIVE_DOUBLE); bandParamsType.insertMember("LOW_FREQ", HOFFSET(BandParams, lowFrequency), H5::PredType::NATIVE_DOUBLE); bandParamsType.insertMember("HIGH_FREQ", HOFFSET(BandParams, highFrequency), H5::PredType::NATIVE_DOUBLE); bandParamsType.insertMember("N_CHANS", HOFFSET(BandParams, nChannels), H5::PredType::NATIVE_INT); bandParamsType.insertMember("N_POLS", HOFFSET(BandParams, nPolarizations), H5::PredType::NATIVE_INT); _telescopeName = header.telescopeName; Logger::Debug << "Opening sdhdf file for telescope " << header.telescopeName << " with " << _beams.size() << " beam(s).\n"; for (size_t beamIndex = 0; beamIndex != _beams.size(); ++beamIndex) { const H5::DataSet beamSet = file.openDataSet( "beam_" + std::to_string(beamIndex) + "/metadata/band_params"); const H5::DataSpace space = beamSet.getSpace(); std::vector dims(space.getSimpleExtentNdims()); if (dims.size() != 1) throw std::runtime_error("Invalid dimensionality of band_params"); space.getSimpleExtentDims(dims.data()); const size_t nBands = dims[0]; Beam& beam = _beams[beamIndex]; beam.bandParams.resize(nBands); beamSet.read(beam.bandParams.data(), bandParamsType); for (size_t bandIndex = 0; bandIndex != nBands; ++bandIndex) _indexTable.emplace_back(beamIndex, bandIndex); } } SdhdfImageSet::~SdhdfImageSet() {} void SdhdfImageSet::Initialize() {} std::string SdhdfImageSet::Description(const ImageSetIndex& index) const { const size_t beam = _indexTable[index.Value()].first; const size_t band = _indexTable[index.Value()].second; return "Beam " + std::to_string(beam) + ", " + _beams[beam].bandParams[band].label; } bool SdhdfImageSet::tryOpen(H5::DataSet& dataSet, H5::H5File& file, const std::string& name) { #if H5_VERSION_GE(1, 10, 7) if (file.exists(name)) { dataSet = file.openDataSet(name); return true; } else { return false; } #else // file.exists(name) doesn't work in older HDF5 libs, so use trial // and error. Unfortunately this generates output on the cmdline // when the file does not exist, hence only use it as fall back. try { dataSet = file.openDataSet(name); return true; } catch (H5::FileIException&) { // The data set did not exist (probably) return false; } #endif } BaselineData SdhdfImageSet::loadData(ProgressListener& progress, const ImageSetIndex& index) { const size_t beam = _indexTable[index.Value()].first; const size_t band = _indexTable[index.Value()].second; const BandParams& bandParams = _beams[beam].bandParams[band]; const std::string label = bandParams.label; Logger::Info << "Reading beam " << beam << ", " << label << "...\n"; H5::H5File file(_path, H5F_ACC_RDONLY); const std::string beamPrefix = "beam_" + std::to_string(beam) + "/" + label; const std::string dataPrefix = beamPrefix + "/astronomy_data/"; const H5::DataSet dataSet = file.openDataSet(dataPrefix + "data"); const H5::DataSpace space = dataSet.getSpace(); std::vector ddims(space.getSimpleExtentNdims()); space.getSimpleExtentDims(ddims.data()); // should be time, beam, polarization, frequency, bin if (ddims.size() != 5) throw std::runtime_error("Incorrect dimensionality of main data array"); const size_t nTimes = ddims[0]; if (ddims[1] != 1) throw std::runtime_error( "Beam axis size is not one, don't know how to handle this file"); if (int(ddims[2]) != bandParams.nPolarizations) throw std::runtime_error("Polarization axis has incorrect size"); if (int(ddims[3]) != bandParams.nChannels) throw std::runtime_error("Frequency axis has incorrect size"); if (ddims[4] != 1) throw std::runtime_error( "bin axis size is not one, don't know how to handle this file"); aocommon::UVector data(nTimes * bandParams.nPolarizations * bandParams.nChannels); progress.OnProgress(1, 10); dataSet.read(data.data(), H5::PredType::NATIVE_FLOAT); std::vector images(bandParams.nPolarizations); for (int pol = 0; pol != bandParams.nPolarizations; ++pol) images[pol] = Image2D::CreateUnsetImagePtr(nTimes, bandParams.nChannels); const float* valuePtr = data.data(); for (size_t timeIndex = 0; timeIndex != nTimes; ++timeIndex) { for (int pol = 0; pol != bandParams.nPolarizations; ++pol) { for (int channel = 0; channel != bandParams.nChannels; ++channel) { images[pol]->SetValue(timeIndex, channel, *valuePtr); ++valuePtr; } } } Mask2DPtr mask; H5::DataSet flagsSet; if (tryOpen(flagsSet, file, dataPrefix + "flag")) { progress.OnProgress(5, 10); Logger::Info << "Reading flags...\n"; const H5::DataSpace space = flagsSet.getSpace(); std::vector fdims(space.getSimpleExtentNdims()); space.getSimpleExtentDims(fdims.data()); // should be time, frequency if (fdims.size() != 2) throw std::runtime_error("Incorrect dimensionality of main flag array"); const size_t nTimes = fdims[0]; if (size_t(fdims[0]) != nTimes) throw std::runtime_error( "Polarization axis of flag data has incorrect size"); if (int(fdims[1]) != bandParams.nChannels) throw std::runtime_error( "Frequency axis of flag data has incorrect size"); aocommon::UVector flags(nTimes * bandParams.nChannels); progress.OnProgress(6, 10); flagsSet.read(flags.data(), H5::PredType::NATIVE_UINT8); mask = Mask2D::CreateUnsetMaskPtr(nTimes, bandParams.nChannels); const unsigned char* valuePtr = flags.data(); for (size_t timeIndex = 0; timeIndex != nTimes; ++timeIndex) { for (int channel = 0; channel != bandParams.nChannels; ++channel) { mask->SetValue(timeIndex, channel, *valuePtr); ++valuePtr; } } } progress.OnProgress(9, 10); TimeFrequencyData tfData; if (bandParams.nPolarizations == 4) tfData = TimeFrequencyData::FromLinear( TimeFrequencyData::AmplitudePart, std::move(images[0]), std::move(images[2]), std::move(images[3]), std::move(images[1])); else if (bandParams.nPolarizations == 2) tfData = TimeFrequencyData(TimeFrequencyData::AmplitudePart, aocommon::PolarizationEnum::XX, images[0], aocommon::PolarizationEnum::YY, images[1]); else if (bandParams.nPolarizations == 1) tfData = TimeFrequencyData(TimeFrequencyData::AmplitudePart, aocommon::PolarizationEnum::StokesI, images[0]); else throw std::runtime_error( "Don't know how to convert the polarizations in this data set"); if (mask) tfData.SetGlobalMask(mask); const TimeFrequencyMetaDataPtr metaData(new TimeFrequencyMetaData()); const H5::CompType obsParamsType(sizeof(ObsParams)); obsParamsType.insertMember("MJD", HOFFSET(ObsParams, mjd), H5::PredType::NATIVE_DOUBLE); const H5::DataSet obsParamsSet = file.openDataSet(beamPrefix + "/metadata/obs_params"); const H5::DataSpace obsParamsSpace = obsParamsSet.getSpace(); if (obsParamsSpace.getSimpleExtentNdims() != 1) throw std::runtime_error( "Incorrect dimensionality of metadata/obs_params for this beam/band"); hsize_t obsParamsDims; obsParamsSpace.getSimpleExtentDims(&obsParamsDims); if (obsParamsDims != nTimes) throw std::runtime_error( "Incorrect nr of elements in metadata/obs_params data set for this " "beam/band"); std::vector obsParams(nTimes); obsParamsSet.read(obsParams.data(), obsParamsType); std::vector times; times.reserve(nTimes); for (const ObsParams& obsParamsEl : obsParams) times.emplace_back(obsParamsEl.mjd * 24.0 * 60.0 * 60.0); metaData->SetObservationTimes(std::move(times)); const H5::DataSet frequencySet = file.openDataSet(beamPrefix + "/astronomy_data/frequency"); const H5::DataSpace frequencySpace = frequencySet.getSpace(); if (frequencySpace.getSimpleExtentNdims() != 1) throw std::runtime_error( "Incorrect dimensionality of astronomy_data/frequency for this " "beam/band"); hsize_t frequencySpaceDims; frequencySpace.getSimpleExtentDims(&frequencySpaceDims); if (int(frequencySpaceDims) != bandParams.nChannels) throw std::runtime_error( "Incorrect nr of elements (" + std::to_string(frequencySpaceDims) + ") in astronomy_data/frequency for this beam/band"); std::vector channels(bandParams.nChannels); frequencySet.read(channels.data(), H5::PredType::NATIVE_FLOAT); BandInfo bandInfo; bandInfo.channels.resize(bandParams.nChannels); for (int ch = 0; ch != bandParams.nChannels; ++ch) { bandInfo.channels[ch].frequencyHz = channels[ch] * 1e6; bandInfo.channels[ch].frequencyIndex = ch; } metaData->SetBand(bandInfo); return BaselineData(tfData, metaData, index); } void SdhdfImageSet::PerformReadRequests(ProgressListener& progress) { progress.OnStartTask("Reading " + _path); for (size_t i = 0; i != _requests.size(); ++i) { const ImageSetIndex& index = _requests[i]; SubTaskListener subListener(progress, i, _requests.size()); _baselineBuffer.emplace(loadData(subListener, index)); progress.OnProgress(i + 1, _requests.size()); } progress.OnFinish(); } void SdhdfImageSet::AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) { assert(!flags.empty()); const size_t beam = _indexTable[index.Value()].first; const size_t band = _indexTable[index.Value()].second; const BandParams& bandParams = _beams[beam].bandParams[band]; const std::string label = bandParams.label; const size_t nTimes = flags.front()->Width(); Logger::Info << "Writing flags for beam " << beam << ", " << label << "...\n"; H5::H5File file(_path, H5F_ACC_RDWR); const std::string beamPrefix = "beam_" + std::to_string(beam) + '/' + label + '/'; const std::string dataPrefix = beamPrefix + "astronomy_data/"; H5::DataSet flagsSet; if (!tryOpen(flagsSet, file, dataPrefix + "flag")) { // Create the data set const H5::DataType dataType(H5::PredType::NATIVE_UINT8); const hsize_t dimsf[2]{nTimes, hsize_t(bandParams.nChannels)}; const H5::DataSpace dataSpace(2, dimsf); flagsSet = file.createDataSet(dataPrefix + "flag", dataType, dataSpace); } Logger::Info << "Writing flag data...\n"; aocommon::UVector flagData(nTimes * bandParams.nChannels); unsigned char* valuePtr = flagData.data(); for (size_t timeIndex = 0; timeIndex != nTimes; ++timeIndex) { for (int channel = 0; channel != bandParams.nChannels; ++channel) { bool flag = false; for (const Mask2DCPtr& mask : flags) flag = flag || mask->Value(timeIndex, channel); *valuePtr = flag ? 1 : 0; ++valuePtr; } } flagsSet.write(flagData.data(), H5::PredType::NATIVE_UINT8); } } // namespace imagesets aoflagger-v3.5.1/imagesets/indexableset.cpp0000664000175000017500000000650014752462134017124 0ustar oleole#include "indexableset.h" #include namespace imagesets { bool IndexableSet::FindExtremeBaseline(imagesets::ImageSet* imageSet, imagesets::ImageSetIndex& index, bool longest) { imagesets::IndexableSet* iSet = dynamic_cast(imageSet); if (iSet != nullptr) { double extremeSq = longest ? 0.0 : std::numeric_limits::max(); const size_t antCount = iSet->AntennaCount(); std::vector antennas(antCount); for (size_t a = 0; a != antCount; a++) antennas[a] = iSet->GetAntennaInfo(a); imagesets::ImageSetIndex loopIndex(iSet->StartIndex()); if (loopIndex.HasWrapped()) return false; const size_t band = iSet->GetBand(index), sequenceId = iSet->GetSequenceId(index); while (!loopIndex.HasWrapped()) { if (sequenceId == iSet->GetSequenceId(loopIndex) && band == iSet->GetBand(loopIndex)) { const size_t a1 = iSet->GetAntenna1(loopIndex); const size_t a2 = iSet->GetAntenna2(loopIndex); const AntennaInfo &ant1 = antennas[a1], &ant2 = antennas[a2]; const double distSq = ant1.position.DistanceSquared(ant2.position); if (longest) { if (distSq > extremeSq) { extremeSq = distSq; index = loopIndex; } } else { if (distSq < extremeSq && a1 != a2) { extremeSq = distSq; index = loopIndex; } } } loopIndex.Next(); } if (!longest && extremeSq == std::numeric_limits::max()) return false; else return true; } return false; } bool IndexableSet::FindMedianBaseline(imagesets::ImageSet* imageSet, imagesets::ImageSetIndex& index) { imagesets::IndexableSet* iSet = dynamic_cast(imageSet); if (iSet == nullptr) { return false; } else { const size_t antCount = iSet->AntennaCount(); std::vector antennas(antCount); for (size_t a = 0; a != antCount; a++) antennas[a] = iSet->GetAntennaInfo(a); imagesets::ImageSetIndex loopIndex(iSet->StartIndex()); const size_t band = iSet->GetBand(index), sequenceId = iSet->GetSequenceId(index); std::vector> distances; while (!loopIndex.HasWrapped()) { if (sequenceId == iSet->GetSequenceId(loopIndex) && band == iSet->GetBand(loopIndex)) { const size_t a1 = iSet->GetAntenna1(loopIndex); const size_t a2 = iSet->GetAntenna2(loopIndex); const AntennaInfo &ant1 = antennas[a1], &ant2 = antennas[a2]; distances.emplace_back(ant1.position.DistanceSquared(ant2.position), loopIndex); } loopIndex.Next(); } const size_t n = distances.size(); if (n == 0) { return false; } else { std::nth_element( distances.begin(), distances.begin() + n / 2, distances.end(), [](const std::pair& l, const std::pair& r) { return l.first < r.first; }); index = (distances.begin() + n / 2)->second; return true; } } } } // namespace imagesets aoflagger-v3.5.1/imagesets/h5imageset.cpp0000664000175000017500000002023714752464167016523 0ustar oleole#include "h5imageset.h" #include "../util/logger.h" #include "../util/progress/subtasklistener.h" #include #include namespace imagesets { namespace { constexpr char kSubArrayPrefix[] = "SUB_ARRAY_POINTING_"; constexpr char kBeamPrefix[] = "BEAM_"; constexpr char kCoordinate0Path[] = "/COORDINATES/COORDINATE_0"; constexpr char kCoordinate1Path[] = "/COORDINATES/COORDINATE_1"; constexpr char kMainDataPath[] = "/STOKES_0"; constexpr char kIncrementAttribute[] = "INCREMENT"; constexpr char kTelescopeAttribute[] = "TELESCOPE"; constexpr char kFrequencyAxisAttribute[] = "AXIS_VALUES_WORLD"; int AddToVectorOperator(hid_t loc_id, const char* name, void* list) { const H5I_type_t type = H5::IdComponent::getHDFObjType(loc_id); if (type == H5I_type_t::H5I_GROUP) { std::vector& vector_list = *reinterpret_cast*>(list); vector_list.emplace_back(name); } return 0; } std::vector GetChildGroups(H5::H5File& file, const std::string& path) { std::vector list; file.iterateElems(path, nullptr, &AddToVectorOperator, &list); return list; } std::string GetFirstGroupFromPrefix(H5::H5File& file, const std::string& path, const std::string& prefix) { const std::vector list = GetChildGroups(file, path); for (const std::string& name : list) { auto res = std::mismatch(prefix.begin(), prefix.end(), name.begin()); if (res.first == prefix.end()) { return name; } } throw std::runtime_error( "Could not find expected group with prefix " + prefix + " in file: can't interpret this as a LOFAR H5 file."); } /** * Read the INCREMENT HDF5 field, which identified the seconds between * timesteps. */ double GetTimeIncrease(H5::H5File& file, const std::string& prefix_path) { // Read the timestep increase const H5::Group group = file.openGroup(prefix_path + kCoordinate0Path); const H5::Attribute attribute = group.openAttribute(kIncrementAttribute); const H5::DataSpace space = attribute.getSpace(); if (space.getSimpleExtentNdims() != 0) throw std::runtime_error("Invalid nr of dimensions (" + std::to_string(space.getSimpleExtentNdims()) + ") for attribute"); double time_increase; attribute.read(H5::PredType::NATIVE_DOUBLE, &time_increase); return time_increase; } /** * Read the frequency axis, i.e., the frequency of every channel * (normally is uniform, but could theoretically not be the case) */ aocommon::UVector GetFrequencyAxis(H5::H5File& file, const std::string& prefix_path) { const H5::Group group = file.openGroup(prefix_path + kCoordinate1Path); const H5::Attribute attribute = group.openAttribute(kFrequencyAxisAttribute); const H5::DataSpace space = attribute.getSpace(); if (space.getSimpleExtentNdims() != 1) throw std::runtime_error("Invalid nr of dimensions (" + std::to_string(space.getSimpleExtentNdims()) + ") for attribute"); hsize_t freq_axis_size; space.getSimpleExtentDims(&freq_axis_size); aocommon::UVector frequency_axis(freq_axis_size); attribute.read(H5::PredType::NATIVE_DOUBLE, frequency_axis.data()); return frequency_axis; } /** * Read the "STOKES" field, transpose the data and return it * in a Image2D. An optional interval start and end can be specified * to limit reading from and/or up to some timestep. If either * one is not specified, the first and last timestep are used, * respectively. * @param range_start is the index of the first timestep after * having truncated the requested interval to the available times. */ Image2DPtr GetMainValues(H5::H5File& file, const std::string& prefix_path, size_t& range_start, std::optional interval_start, std::optional interval_end) { const H5::DataSet data_set = file.openDataSet(prefix_path + kMainDataPath); const H5::DataSpace space = data_set.getSpace(); std::vector dims(space.getSimpleExtentNdims()); if (dims.size() != 2) throw std::runtime_error("Invalid dimensionality of data"); space.getSimpleExtentDims(dims.data()); const size_t n_times = dims[0]; const size_t n_channels = dims[1]; range_start = std::min(interval_start.value_or(0), n_times); const size_t range_end = interval_end ? std::min(*interval_end, n_times) : n_times; dims[0] = range_end - range_start; const hsize_t mem_offsets[] = {0, 0}; const H5::DataSpace memory_space(2, dims.data()); memory_space.selectHyperslab(H5S_SELECT_SET, dims.data(), mem_offsets); const hsize_t file_offsets[] = {range_start, 0}; const H5::DataSpace file_space(space); file_space.selectHyperslab(H5S_SELECT_SET, dims.data(), file_offsets); Image2DPtr image = Image2D::CreateUnsetImagePtr(n_channels, range_end - range_start); data_set.read(image->Data(), H5::PredType::NATIVE_FLOAT, memory_space, file_space); // In the H5 format, frequency is the fastest increasing axis. AOFlagger // expects this to be time: image->Transpose(); return image; } } // namespace H5ImageSet::H5ImageSet(const std::string& path) : path_(path) { const H5::H5File file(path_, H5F_ACC_RDONLY); const H5::Attribute attribute = file.openAttribute(kTelescopeAttribute); attribute.read(H5::StrType(0, H5T_VARIABLE), telescope_name_); } void H5ImageSet::Initialize() {} std::string H5ImageSet::Description(const ImageSetIndex&) const { return path_; } std::unique_ptr H5ImageSet::LoadData(ProgressListener&, const ImageSetIndex& index) { H5::H5File file(path_, H5F_ACC_RDONLY); // We look for something like: SUB_ARRAY_POINTING_000/BEAM_000 // However, numbers may change, so the members have to be iterated. const std::string sub_array_path = GetFirstGroupFromPrefix(file, "/", kSubArrayPrefix); const std::string beam_path = sub_array_path + "/" + GetFirstGroupFromPrefix(file, sub_array_path, kBeamPrefix); const double time_increase = GetTimeIncrease(file, beam_path); const aocommon::UVector frequency_axis = GetFrequencyAxis(file, beam_path); size_t range_start = 0; Image2DPtr image = GetMainValues(file, beam_path, range_start, interval_start_, interval_end_); const size_t n_frequencies = image->Height(); const size_t n_times = image->Width(); if (n_frequencies != frequency_axis.size()) throw std::runtime_error("Inconsistent axes in file"); const TimeFrequencyData data(TimeFrequencyData::AmplitudePart, aocommon::PolarizationEnum::StokesI, std::move(image)); // Fill the meta data structure TimeFrequencyMetaDataPtr meta_data(new TimeFrequencyMetaData()); BandInfo band_info; band_info.channels.resize(n_frequencies); for (size_t ch = 0; ch != n_frequencies; ++ch) { band_info.channels[ch].frequencyHz = frequency_axis[ch]; band_info.channels[ch].frequencyIndex = ch; } meta_data->SetBand(band_info); std::vector times; times.reserve(n_times); for (size_t i = 0; i != n_times; ++i) { times.emplace_back(time_increase * (i + range_start)); } meta_data->SetObservationTimes(std::move(times)); return std::make_unique(std::move(data), std::move(meta_data), index); } void H5ImageSet::PerformReadRequests(ProgressListener& progress) { progress.OnStartTask("Reading " + path_); for (size_t i = 0; i != requests_.size(); ++i) { const ImageSetIndex& index = requests_[i]; SubTaskListener sub_listener(progress, i, requests_.size()); baseline_buffer_.emplace(LoadData(sub_listener, index)); progress.OnProgress(i + 1, requests_.size()); } progress.OnFinish(); } void H5ImageSet::AddWriteFlagsTask(const ImageSetIndex&, std::vector&) { throw std::runtime_error( "Flags can not be written, H5 files do not support flags"); } } // namespace imagesets aoflagger-v3.5.1/imagesets/imagesetindex.h0000664000175000017500000000143214752462134016747 0ustar oleole#ifndef RFI_STRATEGY_IMAGESETINDEX_H #define RFI_STRATEGY_IMAGESETINDEX_H #include namespace imagesets { class ImageSet; class ImageSetIndex final { public: ImageSetIndex() : _n(0), _value(0), _hasWrapped(true) {} ImageSetIndex(size_t n, size_t value = 0) : _n(n), _value(value), _hasWrapped(false) {} void Previous() { if (_value == 0) { _value = _n - 1; _hasWrapped = true; } else { --_value; } } bool Empty() const { return _n == 0; } void Next() { ++_value; if (_value == _n) { _hasWrapped = true; _value = 0; } } size_t Value() const { return _value; } bool HasWrapped() const { return _hasWrapped; } private: size_t _n, _value; bool _hasWrapped; }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/fitsimageset.cpp0000664000175000017500000006627314752462134017156 0ustar oleole#include "fitsimageset.h" #include #include #include "../structures/date.h" #include "../msio/fitsfile.h" #include "../structures/image2d.h" #include "../structures/timefrequencydata.h" #include "../structures/timefrequencymetadata.h" #include "../util/logger.h" namespace imagesets { FitsImageSet::FitsImageSet(const std::string& file) : ImageSet(), _file(new FitsFile(file)), _currentBaselineIndex(0), _frequencyOffset(0.0), _fitsType(UVFitsType) { _file->Open(FitsFile::ReadWriteMode); } FitsImageSet::FitsImageSet(const FitsImageSet& source) : ImageSet(), _file(source._file), _baselines(source._baselines), _bandCount(source._bandCount), _antennaInfos(source._antennaInfos), _bandInfos(source._bandInfos), _bandIndexToNumber(source._bandIndexToNumber), _currentBaselineIndex(source._currentBaselineIndex), _currentBandIndex(source._currentBandIndex), _frequencyOffset(source._frequencyOffset), _baselineData(source._baselineData), _fitsType(source._fitsType) {} FitsImageSet::~FitsImageSet() {} std::unique_ptr FitsImageSet::Clone() { return std::unique_ptr(new FitsImageSet(*this)); } void FitsImageSet::Initialize() { if (_file->HasGroups()) _fitsType = UVFitsType; else if (_file->GetHDUCount() > 1) _fitsType = SDFitsType; else _fitsType = DynSpectrumType; switch (_fitsType) { case UVFitsType: { Logger::Debug << "This file has " << _file->GetGroupCount() << " groups with " << _file->GetParameterCount() << " parameters.\n"; _file->MoveToHDU(1); if (_file->GetCurrentHDUType() != FitsFile::ImageHDUType) throw FitsIOException("Primary table is not a grouped image"); std::vector parameters(_file->GetParameterCount()); const int baselineIndex = _file->GetGroupParameterIndex("BASELINE"); const size_t groupCount = _file->GetGroupCount(); std::set> baselineSet; for (size_t g = 0; g < groupCount; ++g) { _file->ReadGroupParameters(g, parameters.data()); const int a1 = (((int)parameters[baselineIndex]) & 255) - 1; const int a2 = (((int)parameters[baselineIndex] >> 8) & 255) - 1; baselineSet.insert(std::pair(a1, a2)); } Logger::Debug << "Baselines in file: " << baselineSet.size() << '\n'; for (std::set>::const_iterator i = baselineSet.begin(); i != baselineSet.end(); ++i) _baselines.push_back(*i); _bandCount = _file->GetCurrentImageSize(5); } break; case SDFitsType: { _baselines.push_back(std::pair(0, 0)); AntennaInfo antenna; antenna.id = 0; _antennaInfos.push_back(antenna); // find number of bands _file->MoveToHDU(2); int ifColumn = 0; const bool hasIF = _file->HasTableColumn("IF", ifColumn); if (!hasIF) ifColumn = _file->GetTableColumnIndex("IFNUM"); const int rowCount = _file->GetRowCount(); std::set ifSet; for (int i = 1; i <= rowCount; ++i) { double thisIndex; _file->ReadTableCell(i, ifColumn, &thisIndex, 1); ifSet.insert((int)round(thisIndex)); } _bandCount = ifSet.size(); if (_bandCount == 0) throw std::runtime_error("Could not find any IF's in this set"); _bandIndexToNumber.clear(); Logger::Debug << _bandCount << " IF's in set: [" << *ifSet.begin(); for (const int i : ifSet) { _bandInfos.emplace(i, BandInfo()); if (_bandIndexToNumber.size() > 0) Logger::Debug << ", " << i; _bandIndexToNumber.push_back(i); } Logger::Debug << "]\n"; } break; case DynSpectrumType: _baselines.push_back(std::pair(0, 0)); AntennaInfo antenna; antenna.id = 0; antenna.name = ""; const size_t height = _file->GetCurrentImageSize(2); _antennaInfos.emplace_back(antenna); _bandCount = 1; _bandInfos.emplace(0, BandInfo()); _bandInfos[0].channels.resize(height); const double freq0 = _file->GetDoubleKeywordValue("CRVAL2"); const double freqDelta = _file->GetDoubleKeywordValue("CDELT2"); _sourceName = _file->GetKeywordValue("SOURCE"); for (size_t i = 0; i != _bandInfos[0].channels.size(); ++i) { _bandInfos[0].channels[i].frequencyHz = freq0 + i * freqDelta; _bandInfos[0].channels[i].frequencyIndex = i; } _file->MoveToHDU(1); break; } } BaselineData FitsImageSet::loadData(const ImageSetIndex& index) { const size_t baselineIndex = index.Value() / _bandCount; const size_t bandIndex = index.Value() % _bandCount; _frequencyOffset = 0.0; _file->MoveToHDU(1); const TimeFrequencyMetaDataPtr metaData(new TimeFrequencyMetaData()); TimeFrequencyData data; switch (_fitsType) { case UVFitsType: data = ReadPrimaryGroupTable(baselineIndex, bandIndex, 0, *metaData); break; case SDFitsType: ReadPrimarySingleTable(data, *metaData); break; case DynSpectrumType: ReadDynSpectrum(data, *metaData); return BaselineData(data, metaData, index); } for (int hduIndex = 2; hduIndex <= _file->GetHDUCount(); hduIndex++) { _file->MoveToHDU(hduIndex); switch (_file->GetCurrentHDUType()) { case FitsFile::BinaryTableHDUType: Logger::Debug << "Binary table found.\n"; ReadTable(data, *metaData, bandIndex); break; case FitsFile::ASCIITableHDUType: Logger::Debug << "ASCII table found.\n"; ReadTable(data, *metaData, bandIndex); break; case FitsFile::ImageHDUType: Logger::Debug << "Image found.\n"; break; } } if (_fitsType == UVFitsType) { _currentBaselineIndex = baselineIndex; _currentBandIndex = bandIndex; const int bandNumber = _bandIndexToNumber[bandIndex]; metaData->SetBand(_bandInfos[bandNumber]); metaData->SetAntenna1( _antennaInfos[_baselines[_currentBaselineIndex].first]); metaData->SetAntenna2( _antennaInfos[_baselines[_currentBaselineIndex].second]); Logger::Debug << "Loaded metadata for: " << Date::AipsMJDToString(metaData->ObservationTimes()[0]) << ", band " << bandNumber << " (" << Frequency::ToString( _bandInfos[bandNumber].channels[0].frequencyHz) << " - " << Frequency::ToString( _bandInfos[bandNumber].channels.rbegin()->frequencyHz) << ")\n"; } else { metaData->SetAntenna1(_antennaInfos[0]); metaData->SetAntenna2(_antennaInfos[0]); } return BaselineData(data, metaData, index); } TimeFrequencyData FitsImageSet::ReadPrimaryGroupTable( size_t baselineIndex, int band, int stokes, TimeFrequencyMetaData& metaData) { if (!_file->HasGroups() || _file->GetCurrentHDUType() != FitsFile::ImageHDUType) throw FitsIOException("Primary table is not a grouped image"); std::vector observationTimes; std::vector uvws; std::vector parameters(_file->GetParameterCount()); const int baseline = (_baselines[baselineIndex].first + 1) + ((_baselines[baselineIndex].second + 1) << 8); const int baselineColumn = _file->GetGroupParameterIndex("BASELINE"); size_t complexCount = _file->GetCurrentImageSize(2), stokesStep = complexCount, stokesCount = _file->GetCurrentImageSize(3), frequencyStep = stokesCount * complexCount, frequencyCount = _file->GetCurrentImageSize(4), bandStep = frequencyStep * frequencyCount; std::vector> valuesR(frequencyCount); std::vector> valuesI(frequencyCount); std::vector data(_file->GetImageSize()); const size_t groupCount = _file->GetGroupCount(); const bool hasDate2 = _file->HasGroupParameter("DATE", 2); int date2Index = 0, date1Index = _file->GetGroupParameterIndex("DATE"); if (hasDate2) { date2Index = _file->GetGroupParameterIndex("DATE", 2); } int uuIndex, vvIndex, wwIndex; if (_file->HasGroupParameter("UU")) { uuIndex = _file->GetGroupParameterIndex("UU"); vvIndex = _file->GetGroupParameterIndex("VV"); wwIndex = _file->GetGroupParameterIndex("WW"); } else { uuIndex = _file->GetGroupParameterIndex("UU---SIN"); vvIndex = _file->GetGroupParameterIndex("VV---SIN"); wwIndex = _file->GetGroupParameterIndex("WW---SIN"); } size_t match = 0; double frequencyFactor = 1.0; if (_frequencyOffset != 0.0) frequencyFactor = _frequencyOffset; for (size_t g = 0; g < groupCount; ++g) { _file->ReadGroupParameters(g, ¶meters[0]); if (parameters[baselineColumn] == baseline) { double date; if (hasDate2) date = parameters[date1Index] + parameters[date2Index]; else date = parameters[date1Index]; UVW uvw; uvw.u = parameters[uuIndex] * frequencyFactor; uvw.v = parameters[vvIndex] * frequencyFactor; uvw.w = parameters[wwIndex] * frequencyFactor; _file->ReadGroupData(g, &data[0]); for (size_t f = 0; f < frequencyCount; ++f) { const size_t index = stokes * stokesStep + frequencyStep * f + bandStep * band; const long double r = data[index]; const long double i = data[index + 1]; valuesR[f].push_back(r); valuesI[f].push_back(i); } observationTimes.push_back(Date::JDToAipsMJD(date)); uvws.push_back(uvw); ++match; } } Logger::Debug << match << " rows in table matched baseline.\n"; data.clear(); parameters.clear(); Logger::Debug << "Image is " << valuesR[0].size() << " x " << frequencyCount << '\n'; if (valuesR[0].size() == 0) throw std::runtime_error("Baseline not found!"); Image2DPtr real = Image2D::CreateUnsetImagePtr(valuesR[0].size(), frequencyCount), imaginary = Image2D::CreateUnsetImagePtr(valuesR[0].size(), frequencyCount); for (size_t i = 0; i < valuesR[0].size(); ++i) { for (size_t f = 0; f < frequencyCount; ++f) { real->SetValue(i, f, valuesR[f][i]); imaginary->SetValue(i, f, valuesI[f][i]); } } metaData.SetUVW(uvws); metaData.SetObservationTimes(observationTimes); return TimeFrequencyData(aocommon::Polarization::StokesI, real, imaginary); } void FitsImageSet::ReadPrimarySingleTable(TimeFrequencyData& data, TimeFrequencyMetaData& metaData) {} void FitsImageSet::ReadTable(TimeFrequencyData& data, TimeFrequencyMetaData& metaData, size_t bandIndex) { const std::string extName = _file->GetKeywordValue("EXTNAME"); if (extName == "AIPS AN") ReadAntennaTable(metaData); else if (extName == "AIPS FQ") ReadFrequencyTable(data, metaData); else if (extName == "AIPS CL") ReadCalibrationTable(); else if (extName == "SINGLE DISH") ReadSingleDishTable(data, metaData, bandIndex); } void FitsImageSet::ReadAntennaTable(TimeFrequencyMetaData& metaData) { Logger::Debug << "Found antenna table\n"; _frequencyOffset = _file->GetDoubleKeywordValue("FREQ"); for (std::map::iterator i = _bandInfos.begin(); i != _bandInfos.end(); ++i) { for (std::vector::iterator j = i->second.channels.begin(); j != i->second.channels.end(); ++j) { j->frequencyHz += _frequencyOffset; } } std::vector uvws(metaData.UVW()); for (std::vector::iterator i = uvws.begin(); i != uvws.end(); ++i) { i->u = i->u * _frequencyOffset; i->v = i->v * _frequencyOffset; i->w = i->w * _frequencyOffset; } metaData.SetUVW(uvws); _antennaInfos.clear(); for (int i = 1; i <= _file->GetRowCount(); ++i) { AntennaInfo info; char name[9]; long double pos[3]; _file->ReadTableCell(i, 1, name); _file->ReadTableCell(i, 2, pos, 3); info.name = name; info.id = _antennaInfos.size(); info.position.x = pos[0]; info.position.y = pos[1]; info.position.z = pos[2]; _antennaInfos.push_back(info); } } void FitsImageSet::ReadFrequencyTable(TimeFrequencyData& data, TimeFrequencyMetaData& metaData) { Logger::Debug << "Found frequency table\n"; const size_t numberIfs = _file->GetIntKeywordValue("NO_IF"); Logger::Debug << "Number of ifs: " << numberIfs << '\n'; _bandInfos.clear(); BandInfo bandInfo; for (int i = 1; i <= _file->GetRowCount(); ++i) { long double freqSel; std::vector ifFreq(numberIfs), chWidth(numberIfs), totalBandwidth(numberIfs), sideband(numberIfs); _file->ReadTableCell(i, 1, &freqSel, 1); _file->ReadTableCell(i, 2, &ifFreq[0], numberIfs); _file->ReadTableCell(i, 3, &chWidth[0], numberIfs); _file->ReadTableCell(i, 4, &totalBandwidth[0], numberIfs); _file->ReadTableCell(i, 5, &sideband[0], numberIfs); for (size_t b = 0; b < numberIfs; ++b) { for (size_t channel = 0; channel < data.ImageHeight(); ++channel) { ChannelInfo channelInfo; channelInfo.channelWidthHz = chWidth[b]; channelInfo.effectiveBandWidthHz = chWidth[b]; channelInfo.frequencyHz = _frequencyOffset + ifFreq[b] + (chWidth[b] * channel); channelInfo.frequencyIndex = channel; channelInfo.resolutionHz = chWidth[b]; bandInfo.channels.push_back(channelInfo); } bandInfo.windowIndex = b; _bandInfos.insert(std::pair(b, bandInfo)); } } } void FitsImageSet::ReadCalibrationTable() { Logger::Debug << "Found calibration table with " << _file->GetRowCount() << " rows.\n"; } void FitsImageSet::ReadDynSpectrum(TimeFrequencyData& data, TimeFrequencyMetaData& metaData) { _file->MoveToHDU(1); const size_t width = _file->GetCurrentImageSize(1); const size_t height = _file->GetCurrentImageSize(2); const size_t npol = _file->GetCurrentImageSize(3); Logger::Debug << "Reading fits file with dynspectrum, " << width << " x " << height << " x " << npol << "\n"; if (npol != 4) throw std::runtime_error( "Expected four polarizations in dynamic spectrum fits file"); const size_t n = width * height * npol; std::vector buffer(n); _file->ReadCurrentImageData(0, buffer.data(), n); Image2DPtr imgs[4]; for (size_t i = 0; i != 4; ++i) imgs[i] = Image2D::CreateUnsetImagePtr(width, height); const Mask2DPtr flags = Mask2D::CreateSetMask(width, height); std::vector::const_iterator bufferIter = buffer.begin(); for (size_t j = 0; j != npol; ++j) { for (size_t y = 0; y != height; ++y) { for (size_t x = 0; x != width; ++x) { imgs[j]->SetValue(x, y, *bufferIter); if (!std::isfinite(*bufferIter)) flags->SetValue(x, y, true); ++bufferIter; } } } data = TimeFrequencyData::MakeFromPolarizationCombination( TimeFrequencyData(TimeFrequencyData::RealPart, aocommon::Polarization::StokesI, imgs[0]), TimeFrequencyData(TimeFrequencyData::RealPart, aocommon::Polarization::StokesQ, imgs[1]), TimeFrequencyData(TimeFrequencyData::RealPart, aocommon::Polarization::StokesU, imgs[2]), TimeFrequencyData(TimeFrequencyData::RealPart, aocommon::Polarization::StokesV, imgs[3])); data.SetGlobalMask(flags); metaData.SetBand(_bandInfos[0]); metaData.SetAntenna1(_antennaInfos[0]); metaData.SetAntenna2(_antennaInfos[0]); std::vector times(width); const double timeDelta = _file->GetDoubleKeywordValue("CDELT1"); for (size_t i = 0; i != width; ++i) times[i] = timeDelta * i; metaData.SetObservationTimes(times); } void FitsImageSet::ReadSingleDishTable(TimeFrequencyData& data, TimeFrequencyMetaData& metaData, size_t ifIndex) { const int rowCount = _file->GetRowCount(); Logger::Debug << "Found single dish table with " << rowCount << " rows.\n"; const int dateObsColumn = _file->GetTableColumnIndex("DATE-OBS"), dataColumn = _file->GetTableColumnIndex("DATA"), freqValColumn = _file->GetTableColumnIndex("CRVAL1"), freqRefPixColumn = _file->GetTableColumnIndex("CRPIX1"), freqDeltaColumn = _file->GetTableColumnIndex("CDELT1"), freqResColumn = _file->GetTableColumnIndex("FREQRES"), freqBandwidthColumn = _file->GetTableColumnIndex("BANDWID"); int timeColumn; const bool hasTime = _file->HasTableColumn("TIME", timeColumn); // optional int flagColumn; const bool hasFlags = _file->HasTableColumn("FLAGGED", flagColumn); // optional int ifColumn; const bool hasIF = _file->HasTableColumn("IF", ifColumn); if (!hasIF) ifColumn = _file->GetTableColumnIndex("IFNUM"); std::vector axisDims = _file->GetColumnDimensions(dataColumn); int freqCount = 0, polarizationCount = 0, raCount = 0, decCount = 0; for (size_t i = 0; i != axisDims.size(); ++i) { const std::string name = _file->GetTableDimensionName(i); if (name == "FREQ") freqCount = axisDims[i]; else if (name == "STOKES") polarizationCount = axisDims[i]; else if (name == "RA") raCount = axisDims[i]; else if (name == "DEC") decCount = axisDims[i]; } const int totalSize = _file->GetTableColumnArraySize(dataColumn); if (freqCount == 0) { freqCount = totalSize; polarizationCount = 1; raCount = 1; decCount = 1; } const std::string telescopeName = _file->GetKeywordValue("TELESCOP"); _antennaInfos[0].name = telescopeName; Logger::Debug << "Shape of data cells: " << freqCount << " channels x " << polarizationCount << " pols x " << raCount << " RAs x " << decCount << " decs" << "=" << totalSize << '\n'; std::vector cellData(totalSize); std::unique_ptr flagData(new bool[totalSize]); for (int i = 0; i != totalSize; ++i) flagData[i] = false; std::vector images(polarizationCount); std::vector masks(polarizationCount); for (int i = 0; i < polarizationCount; ++i) { images[i] = Image2D::CreateZeroImagePtr(rowCount, freqCount); masks[i] = Mask2D::CreateSetMaskPtr(rowCount, freqCount); } std::vector observationTimes(rowCount); bool hasBand = false; const int requestedIFNumber = _bandIndexToNumber[ifIndex]; size_t timeIndex = 0; for (int row = 1; row <= rowCount; ++row) { long double time, date, ifNumber; _file->ReadTableCell(row, ifColumn, &ifNumber, 1); if (ifNumber == requestedIFNumber) { if (hasTime) { _file->ReadTableCell(row, timeColumn, &time, 1); observationTimes[timeIndex] = time; } _file->ReadTableCell(row, dateObsColumn, &date, 1); _file->ReadTableCell(row, dataColumn, &cellData[0], totalSize); if (hasFlags) _file->ReadTableCell(row, flagColumn, &flagData[0], totalSize); if (!hasBand) { long double freqVal = 0.0, freqRefPix = 0.0, freqDelta = 0.0, freqRes = 0.0, freqBandwidth = 0.0; _file->ReadTableCell(row, freqValColumn, &freqVal, 1); _file->ReadTableCell(row, freqRefPixColumn, &freqRefPix, 1); _file->ReadTableCell(row, freqDeltaColumn, &freqDelta, 1); _file->ReadTableCell(row, freqResColumn, &freqRes, 1); _file->ReadTableCell(row, freqBandwidthColumn, &freqBandwidth, 1); if (freqBandwidth > 0.0) { Logger::Debug << "Frequency info: " << freqVal << " Hz at index " << freqRefPix << ", delta " << freqDelta << "\n"; Logger::Debug << "Frequency res: " << freqRes << " with bandwidth " << freqBandwidth << " Hz\n"; BandInfo bandInfo; bandInfo.windowIndex = ifNumber; for (int i = 0; i < freqCount; ++i) { ChannelInfo c; c.frequencyIndex = i; c.frequencyHz = ((double)i - freqRefPix) * freqDelta + freqVal; bandInfo.channels.push_back(c); } _bandInfos[ifNumber] = bandInfo; metaData.SetBand(bandInfo); hasBand = true; } } long double* dataPtr = &cellData[0]; bool* flagPtr = flagData.get(); for (int p = 0; p < polarizationCount; ++p) { for (int f = 0; f < freqCount; ++f) { images[p]->SetValue(timeIndex, f, *dataPtr); masks[p]->SetValue(timeIndex, f, *flagPtr); ++dataPtr; ++flagPtr; } } ++timeIndex; } } flagData.reset(); if (timeIndex == 0) { throw std::runtime_error( "Couldn't find any rows in the fits image set for the requested IF"); } for (int p = 0; p < polarizationCount; ++p) { images[p]->SetTrim(0, 0, timeIndex, images[p]->Height()); masks[p].reset( new Mask2D(masks[p]->Trim(0, 0, timeIndex, images[p]->Height()))); } if (hasTime) { observationTimes.resize(timeIndex); metaData.SetObservationTimes(observationTimes); } if (polarizationCount == 1) { data = TimeFrequencyData(TimeFrequencyData::AmplitudePart, aocommon::Polarization::StokesI, images[0]); data.SetGlobalMask(masks[0]); } else if (polarizationCount == 2) { data = TimeFrequencyData(TimeFrequencyData::AmplitudePart, aocommon::Polarization::XX, images[0], aocommon::Polarization::YY, images[1]); data.SetIndividualPolarizationMasks(masks[0], masks[1]); } else { std::ostringstream s; s << "SDFits file has " << polarizationCount << " polarizations: don't know how to convert these"; throw std::runtime_error(s.str()); } } void FitsImageSet::AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) { switch (_fitsType) { case UVFitsType: throw std::runtime_error("Not implemented for UV fits files"); case SDFitsType: saveSingleDishFlags(flags, index.Value() % _bandCount); break; case DynSpectrumType: saveDynSpectrumFlags(flags); break; } } void FitsImageSet::PerformWriteFlagsTask() { switch (_fitsType) { case UVFitsType: throw std::runtime_error( "Writing flags not implemented for UV fits files"); case SDFitsType: case DynSpectrumType: // Nothing to be done; Add..Task already wrote the flags. break; } } void FitsImageSet::saveSingleDishFlags(const std::vector& flags, size_t ifIndex) { _file->Close(); _file->Open(FitsFile::ReadWriteMode); _file->MoveToHDU(2); Logger::Debug << "Writing single dish table for band " << ifIndex << " with " << _file->GetRowCount() << " rows.\n"; const int dataColumn = _file->GetTableColumnIndex("DATA"), flagColumn = _file->GetTableColumnIndex("FLAGGED"); int ifColumn = 0; const bool hasIF = _file->HasTableColumn("IF", ifColumn); if (!hasIF) ifColumn = _file->GetTableColumnIndex("IFNUM"); const int freqCount = _file->GetColumnDimensionSize(dataColumn, 0), polarizationCount = _file->GetColumnDimensionSize(dataColumn, 1); const int totalSize = _file->GetTableColumnArraySize(dataColumn); const int rowCount = _file->GetRowCount(); std::vector cellData(totalSize); const std::unique_ptr flagData(new bool[totalSize]); std::vector storedFlags = flags; if (flags.size() == 1) { while (storedFlags.size() < (unsigned)polarizationCount) storedFlags.push_back(flags[0]); } if (storedFlags.size() != (unsigned)polarizationCount) { std::stringstream s; s << "saveSingleDishFlags() : mismatch in polarization count: the given " "vector contains " << flags.size() << " polarizations, the number of polarizations in the file is " << polarizationCount; throw std::runtime_error(s.str()); } for (std::vector::const_iterator i = storedFlags.begin(); i != storedFlags.end(); ++i) { if ((*i)->Height() != (unsigned)freqCount) throw std::runtime_error( "Frequency count in given mask does not match with the file"); } size_t timeIndex = 0; const int specifiedIFNumber = _bandIndexToNumber[ifIndex]; for (int row = 1; row <= rowCount; ++row) { long double ifNumber; _file->ReadTableCell(row, ifColumn, &ifNumber, 1); if (ifNumber == specifiedIFNumber) { Logger::Debug << row << "\n"; _file->ReadTableCell(row, dataColumn, &cellData[0], totalSize); double* dataPtr = &cellData[0]; bool* flagPtr = flagData.get(); for (int p = 0; p < polarizationCount; ++p) { for (int f = 0; f < freqCount; ++f) { if (storedFlags[p]->Value(timeIndex, f)) { *flagPtr = true; *dataPtr = 1e20; } else { *flagPtr = false; } ++dataPtr; ++flagPtr; } } _file->WriteTableCell(row, dataColumn, &cellData[0], totalSize); _file->WriteTableCell(row, flagColumn, flagData.get(), totalSize); ++timeIndex; } } } void FitsImageSet::saveDynSpectrumFlags(const std::vector& flags) { Logger::Debug << "Writing dynspectrum flags.\n"; _file->Close(); _file->Open(FitsFile::ReadWriteMode); _file->MoveToHDU(1); const size_t width = _file->GetCurrentImageSize(1); const size_t height = _file->GetCurrentImageSize(2); const size_t npol = _file->GetCurrentImageSize(3); const size_t n = width * height * npol; std::vector buffer(n); _file->ReadCurrentImageData(0, buffer.data(), buffer.size()); std::vector::iterator bufferIter = buffer.begin(); for (size_t j = 0; j != npol; ++j) { for (size_t y = 0; y != height; ++y) { for (size_t x = 0; x != width; ++x) { if (flags[j]->Value(x, y)) *bufferIter = std::numeric_limits::quiet_NaN(); ++bufferIter; } } } _file->WriteImage(0, buffer.data(), buffer.size(), std::numeric_limits::quiet_NaN()); } std::string FitsImageSet::Description(const ImageSetIndex& index) const { if (IsDynSpectrumType()) { return SourceName(); } else { const size_t baselineIndex = index.Value() / _bandCount; const size_t bandIndex = index.Value() % _bandCount; const int a1 = Baselines()[baselineIndex].first; const int a2 = Baselines()[baselineIndex].second; const AntennaInfo info1 = GetAntennaInfo(a1); const AntennaInfo info2 = GetAntennaInfo(a2); std::stringstream s; s << "Correlation " << info1.name << " x " << info2.name << ", band " << bandIndex; return s.str(); } } std::vector FitsImageSet::Files() const { return std::vector{_file->Filename()}; } std::string FitsImageSet::TelescopeName() { if (_fitsType == SDFitsType) { for (int hduIndex = 2; hduIndex <= _file->GetHDUCount(); hduIndex++) { _file->MoveToHDU(hduIndex); const std::string extName = _file->GetKeywordValue("EXTNAME"); if (extName == "SINGLE DISH") return _file->GetKeywordValue("TELESCOP"); } return ""; } else { return "DynSpectrum"; } } } // namespace imagesets aoflagger-v3.5.1/imagesets/indexableset.h0000664000175000017500000000451214752462134016572 0ustar oleole#ifndef INDEXABLE_SET_H #define INDEXABLE_SET_H #include "imageset.h" #include "../msio/baselinereader.h" #include #include namespace imagesets { class IndexableSet : public ImageSet { public: virtual BaselineReaderPtr Reader() const = 0; virtual size_t GetAntenna1(const ImageSetIndex& index) const = 0; virtual size_t GetAntenna2(const ImageSetIndex& index) const = 0; virtual size_t GetBand(const ImageSetIndex& index) const = 0; virtual size_t GetField(const ImageSetIndex& index) const = 0; virtual size_t GetSequenceId(const ImageSetIndex& index) const = 0; virtual size_t AntennaCount() const = 0; virtual AntennaInfo GetAntennaInfo(unsigned antennaIndex) const = 0; virtual size_t BandCount() const = 0; virtual BandInfo GetBandInfo(unsigned bandIndex) const = 0; virtual size_t SequenceCount() const = 0; virtual std::optional Index(size_t antenna1, size_t antenna2, size_t bandIndex, size_t sequenceId) const = 0; virtual FieldInfo GetFieldInfo(unsigned fieldIndex) const = 0; std::string TelescopeName() override; /** * Finds the longest or shortest baseline in the same band/sequence as the * input index. * @param imageSet Input image set * @param index On input, the index that specifies what band/sequence to * search. On output, the found index (if found). * @param longest true for finding the longest, false for finding the shortest * baseline. * @returns true if successful */ static bool FindExtremeBaseline(imagesets::ImageSet* imageSet, imagesets::ImageSetIndex& index, bool longest); /** * Finds the baseline with median length in the same band/sequence as the * input index. * @param imageSet Input image set * @param index On input, the index that specifies what band/sequence to * search. On output, the found index (if found). * @returns true if successful */ static bool FindMedianBaseline(imagesets::ImageSet* imageSet, imagesets::ImageSetIndex& index); }; } // namespace imagesets inline std::string imagesets::IndexableSet::TelescopeName() { auto ms = Reader()->OpenMS(); return MSMetaData::GetTelescopeName(ms); } #endif aoflagger-v3.5.1/imagesets/sdhdfimageset.h0000664000175000017500000000504314752462134016732 0ustar oleole#ifndef SDHDF_IMAGE_SET_H #define SDHDF_IMAGE_SET_H #include #include #include #include #include #include "imageset.h" namespace H5 { class DataSet; class H5File; } // namespace H5 namespace imagesets { /** * Implements the single-dish hdf5 format. * The description of the format is here: * https://bitbucket.csiro.au/projects/CPDA/repos/sdhdf_tools/browse/docs */ class SdhdfImageSet final : public ImageSet { public: SdhdfImageSet(const std::string& path); ~SdhdfImageSet() override; size_t Size() const override { return _indexTable.size(); } std::string Description(const ImageSetIndex& index) const override; std::unique_ptr Clone() override { return std::unique_ptr(new SdhdfImageSet(*this)); } void Initialize() override; std::string Name() const override { return "SDHDF"; } std::vector Files() const override { return std::vector{_path}; } std::string TelescopeName() override { return _telescopeName; } bool HasCrossCorrelations() const override { return false; } void AddReadRequest(const ImageSetIndex& index) override { _requests.emplace_back(index); } void PerformReadRequests(class ProgressListener& progress) override; std::unique_ptr GetNextRequested() override { BaselineData baseline(std::move(_baselineBuffer.front())); _baselineBuffer.pop(); return std::make_unique(baseline); } void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) override; void PerformWriteFlagsTask() override {} private: /** * Open the data set @c name inside @c file, and store the result in @c * dataSet. If the data set does not exist, @c false is returned. */ static bool tryOpen(H5::DataSet& dataSet, H5::H5File& file, const std::string& name); struct Header { int nBeams; char telescopeName[64]; }; struct BandParams { char label[64]; double centreFrequency; double lowFrequency; double highFrequency; int nChannels; int nPolarizations; }; struct Beam { std::vector bandParams; }; const std::string _path; std::vector _requests; std::vector _beams; std::vector> _indexTable; std::queue _baselineBuffer; std::string _telescopeName; BaselineData loadData(ProgressListener& progress, const ImageSetIndex& index); }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/parmimageset.cpp0000664000175000017500000000123014752462134017126 0ustar oleole#include "parmimageset.h" #include #include "../msio/parmtable.h" namespace imagesets { ParmImageSet::~ParmImageSet() { delete _parmTable; } void ParmImageSet::Initialize() { delete _parmTable; _parmTable = new ParmTable(_path); const std::set antennaSet = _parmTable->GetAntennas(); for (std::set::const_iterator i = antennaSet.begin(); i != antennaSet.end(); ++i) _antennas.push_back(*i); } TimeFrequencyData* ParmImageSet::LoadData(const ImageSetIndex& index) { const std::string antenna = _antennas[index.Value()]; return new TimeFrequencyData(_parmTable->Read(antenna)); } } // namespace imagesets aoflagger-v3.5.1/imagesets/rfibaselineset.h0000664000175000017500000000210714752462134017120 0ustar oleole#ifndef RFI_BASELINE_SET_H #define RFI_BASELINE_SET_H #include "../structures/types.h" #include "singleimageset.h" #include #include #include namespace imagesets { class RFIBaselineSet final : public SingleImageSet { public: explicit RFIBaselineSet(const std::string& path); std::unique_ptr Clone() override { return std::unique_ptr(new RFIBaselineSet(*this)); } void Initialize() override {} std::string Name() const override { return _path; } std::string BaselineDescription() override; std::vector Files() const override { return std::vector{_path}; } std::string TelescopeName() override { return _telescopeName; } std::unique_ptr Read(class ProgressListener& progress) override; void Write(const std::vector& masks) override; private: std::string _path; std::string _telescopeName; TimeFrequencyData _data; TimeFrequencyMetaData _metaData; RFIBaselineSet(const RFIBaselineSet& source) = default; }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/msstatset.h0000664000175000017500000000000015136346710016134 0ustar oleoleaoflagger-v3.5.1/imagesets/bhfitsimageset.cpp0000664000175000017500000002254414752462134017461 0ustar oleole#include "bhfitsimageset.h" #include "../msio/fitsfile.h" #include "../structures/image2d.h" #include "../structures/timefrequencydata.h" #include "../util/logger.h" namespace imagesets { BHFitsImageSet::BHFitsImageSet(const std::string& file) : ImageSet(), _file(new FitsFile(file)), _width(0), _height(0) { Logger::Debug << "Opening bhfits file: '" << file << "'\n"; try { _file->Open(FitsFile::ReadWriteMode); } catch (FitsIOException& exception) { Logger::Error << "CFitsio failed to open file in RW mode with the following error:\n" << exception.what() << "\nTrying reopening in read-only mode. Writing to file won't be " "possible.\n\n"; _file->Open(FitsFile::ReadOnlyMode); } } BHFitsImageSet::BHFitsImageSet(const BHFitsImageSet& source) : ImageSet(), _file(source._file), _baselineData(source._baselineData), _timeRanges(source._timeRanges), _width(source._width), _height(source._height) {} BHFitsImageSet::~BHFitsImageSet() = default; std::unique_ptr BHFitsImageSet::Clone() { return std::unique_ptr(new BHFitsImageSet(*this)); } void BHFitsImageSet::Initialize() { _file->MoveToHDU(1); /*for(int i=1;i<=_file->GetKeywordCount();++i) { Logger::Debug << _file->GetKeyword(i) << " = " << _file->GetKeywordValue(i) << '\n'; }*/ if (_file->GetCurrentHDUType() != FitsFile::ImageHDUType) throw std::runtime_error( "Error in Bighorns fits files: first HDU was not an image HDU"); if (_file->GetCurrentImageDimensionCount() != 2) throw std::runtime_error("Fits image was not two dimensional"); _width = _file->GetCurrentImageSize(2), _height = _file->GetCurrentImageSize(1); Logger::Debug << "Image of " << _width << " x " << _height << '\n'; _timeRanges.clear(); size_t keyIndex = 0; bool searchOn; do { searchOn = false; std::ostringstream antKey, termKey; antKey << "ANT"; termKey << "TERM"; if (keyIndex < 10) { antKey << '0'; termKey << '0'; } antKey << keyIndex; termKey << keyIndex; std::string antRangeStr, termRangeStr; if (_file->GetKeywordValue(antKey.str(), antRangeStr)) { const std::pair range = getRangeFromString(antRangeStr); TimeRange timeRange; // As commented by Marcin below, the ranges in the fits headers are given // like 'start - end', where the indices start counting at 1, and the end // index is *inclusive*, such that '1 - 1' represents a range with one // index (being the first timestep in the FITS file). Comment by Marcin // Sokolowski: MS this is due to fact that Andre didn't know it starts // from 1, but he skips the end integration (assumes it is not ANT), but // it is timeRange.start = range.first - 1; // so here I don't subtract 1 in order to program check until this one too // ! timeRange.end = range.second; timeRange.name = antKey.str(); _timeRanges.push_back(timeRange); searchOn = true; } if (_file->GetKeywordValue(termKey.str(), termRangeStr)) { const std::pair range = getRangeFromString(termRangeStr); TimeRange timeRange; // (see earlier comment by Marcin Sokolowski) timeRange.start = range.first - 1; timeRange.end = range.second; timeRange.name = termKey.str(); _timeRanges.push_back(timeRange); searchOn = true; } ++keyIndex; } while (searchOn); Logger::Debug << "This file has " << _timeRanges.size() << " time ranges.\n"; if (_timeRanges.empty()) { // if no states found in the header - just assume all are antenna TimeRange timeRange; // See earlier comment by Marcin Sokolowski timeRange.start = 0; timeRange.end = _file->GetCurrentImageSize(2); timeRange.name = "ANT"; _timeRanges.push_back(timeRange); Logger::Warn << "No states specified in the fits header assuming all (1-" << timeRange.end << ") integrations are " << timeRange.name << "\n"; } } BaselineData BHFitsImageSet::loadData(const ImageSetIndex& index) { const TimeFrequencyMetaDataPtr metaData(new TimeFrequencyMetaData()); TimeFrequencyData data; loadImageData(data, metaData, index); return BaselineData(data, metaData, index); } void BHFitsImageSet::loadImageData(TimeFrequencyData& data, const TimeFrequencyMetaDataPtr& metaData, const ImageSetIndex& index) { std::vector buffer(_width * _height); _file->ReadCurrentImageData(0, &buffer[0], _width * _height); int rangeStart = _timeRanges[index.Value()].start, rangeEnd = _timeRanges[index.Value()].end; const Image2DPtr image = Image2D::CreateZeroImagePtr(rangeEnd - rangeStart, _height); std::vector::const_iterator bufferPtr = buffer.begin() + _height * rangeStart; for (int x = rangeStart; x != rangeEnd; ++x) { for (int y = 0; y != _height; ++y) { image->SetValue(x - rangeStart, y, *bufferPtr); ++bufferPtr; } } data = TimeFrequencyData(TimeFrequencyData::AmplitudePart, aocommon::Polarization::StokesI, image); try { FitsFile flagFile(flagFilePath()); flagFile.Open(FitsFile::ReadOnlyMode); flagFile.ReadCurrentImageData(0, &buffer[0], _width * _height); bufferPtr = buffer.begin() + _height * rangeStart; const Mask2DPtr mask = Mask2D::CreateUnsetMaskPtr(rangeEnd - rangeStart, _height); for (int x = rangeStart; x != rangeEnd; ++x) { for (int y = 0; y != _height; ++y) { bool flag = false; if (*bufferPtr == 0.0) flag = false; else if (*bufferPtr == 1.0) flag = true; else std::runtime_error( "Expecting a flag file with only ones and zeros, but this file " "contained other values."); mask->SetValue(x - rangeStart, y, flag); ++bufferPtr; } } data.SetGlobalMask(mask); } catch (std::exception&) { // Flag file could not be read; probably does not exist. Ignore this, flags // will be initialized to false. } double frequencyDelta = _file->GetDoubleKeywordValue("CDELT1"), timeDelta = _file->GetDoubleKeywordValue("CDELT2"); BandInfo band; for (int ch = 0; ch != _height; ++ch) { ChannelInfo channel; channel.frequencyHz = ch * frequencyDelta * 1000000.0; band.channels.push_back(channel); } metaData->SetBand(band); const int rangeWidth = rangeEnd - rangeStart; std::vector observationTimes(rangeWidth); for (int t = 0; t != rangeWidth; ++t) observationTimes[t] = (t + rangeStart) * timeDelta; metaData->SetObservationTimes(observationTimes); AntennaInfo antennaInfo; antennaInfo.id = 0; antennaInfo.name = RangeName(index.Value()); antennaInfo.diameter = 0.0; antennaInfo.mount = "Unknown"; antennaInfo.station = TelescopeName(); metaData->SetAntenna1(antennaInfo); metaData->SetAntenna2(antennaInfo); } std::pair BHFitsImageSet::getRangeFromString( const std::string& rangeStr) { std::pair value; const size_t partA = rangeStr.find(' '); value.first = atoi(rangeStr.substr(0, partA).c_str()); size_t partB = rangeStr.find('-'); if (rangeStr[partB + 1] == ' ') ++partB; value.second = atoi(rangeStr.substr(partB + 1).c_str()); return value; } std::string BHFitsImageSet::flagFilePath() const { std::string flagFilePath = _file->Filename(); if (flagFilePath.size() > 7) { flagFilePath = flagFilePath.substr(0, flagFilePath.size() - 7); } flagFilePath += "_flag.fits"; return flagFilePath; } void BHFitsImageSet::AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) { if (flags.size() != 1) throw std::runtime_error( "BHFitsImageSet::AddWriteFlagsTask() called with multiple flags"); const std::string flagFilename = flagFilePath(); Logger::Debug << "Writing to " << flagFilename << '\n'; FitsFile flagFile(flagFilename); bool newFile = true; std::vector buffer(_width * _height); try { flagFile.Open(FitsFile::ReadWriteMode); newFile = false; } catch (std::exception&) { Logger::Debug << "File did not exist yet, creating new.\n"; flagFile.Create(); flagFile.AppendImageHUD(FitsFile::Float32ImageType, _height, _width); } // This must be outside the try { } block, so that exceptions // don't result in creating a new file. if (!newFile) { flagFile.ReadCurrentImageData(0, &buffer[0], _width * _height); } int rangeStart = _timeRanges[index.Value()].start, rangeEnd = _timeRanges[index.Value()].end; std::vector::iterator bufferPtr = buffer.begin() + _height * rangeStart; for (int x = rangeStart; x != rangeEnd; ++x) { for (int y = 0; y != _height; ++y) { *bufferPtr = flags[0]->Value(x - rangeStart, y) ? 1.0 : 0.0; ++bufferPtr; } } flagFile.WriteImage(0, &buffer[0], _width * _height, -1.0); } void BHFitsImageSet::PerformWriteFlagsTask() { // Nothing to do; already written } std::string BHFitsImageSet::Description(const ImageSetIndex& index) const { std::ostringstream str; str << "Time range " << RangeName(index.Value()); return str.str(); } std::vector BHFitsImageSet::Files() const { return std::vector{_file->Filename()}; } } // namespace imagesets aoflagger-v3.5.1/imagesets/averagingmsset.h0000664000175000017500000000570615065216451017145 0ustar oleole#ifndef MS_STAT_SET_H #define MS_STAT_SET_H #include #include #include #include #include #include "../msio/averagingmsreader.h" #include "imageset.h" namespace imagesets { class AveragingMsSet final : public ImageSet { public: AveragingMsSet(const std::string& path, const std::string& dataColumn, BaselineIntegrationMode mode, BaselineIntegrationDifferencing diffType, bool includeAutos, bool includeFlags) : path_(path), mode_(mode), differencing_type_(diffType), include_autos_(includeAutos), include_flags_(includeFlags), reader_(std::make_shared( path_, dataColumn.empty() ? "DATA" : dataColumn)) {} std::unique_ptr Clone() final { return std::make_unique(*this); } size_t Size() const final { return reader_->NBands() * reader_->NSequences(); } std::string Description(const ImageSetIndex& index) const final { return Name() + " (over all baselines, " + ToString(mode_) + ", " + ToString(differencing_type_) + ")"; } void Initialize() final {} std::string Name() const final { return path_; } std::vector Files() const final { return std::vector{path_}; } std::string TelescopeName() final { return reader_->TelescopeName(); } void AddReadRequest(const ImageSetIndex& index) final { _requests.emplace_back(index); } void PerformReadRequests(class ProgressListener& progress) final { for (const ImageSetIndex& request : _requests) { size_t bandIndex = request.Value() % reader_->NBands(), seqIndex = request.Value() / reader_->NBands(); AveragingMsReader::Result result = reader_->Read(mode_, differencing_type_, seqIndex, bandIndex, include_autos_, include_flags_, progress); _results.emplace(result.first, result.second, request); } _requests.clear(); } std::unique_ptr GetNextRequested() final { std::unique_ptr result( new BaselineData(std::move(_results.front()))); _results.pop(); return result; } void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) final { size_t bandIndex = index.Value() % reader_->NBands(), seqIndex = index.Value() / reader_->NBands(); reader_->StoreFlags(flags, differencing_type_, seqIndex, bandIndex, include_autos_); } void PerformWriteFlagsTask() final {} bool HasCrossCorrelations() const final { return false; } private: std::string path_; BaselineIntegrationMode mode_; BaselineIntegrationDifferencing differencing_type_; bool include_autos_, include_flags_; std::shared_ptr reader_; std::vector _requests; std::queue _results; }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/bhfitsimageset.h0000664000175000017500000000506114752462134017121 0ustar oleole#ifndef BHFITSIMAGESET_H #define BHFITSIMAGESET_H #include #include #include #include #include #include #include #include "imageset.h" #include "../structures/antennainfo.h" #include "../structures/types.h" namespace imagesets { class BHFitsImageSet final : public ImageSet { public: explicit BHFitsImageSet(const std::string& file); ~BHFitsImageSet(); void Initialize() override; std::unique_ptr Clone() override; std::string Description(const ImageSetIndex& index) const override; std::string Name() const override { return "Bighorns fits file"; } std::vector Files() const override; size_t Size() const override { return _timeRanges.size(); } const std::string& RangeName(size_t rangeIndex) const { return _timeRanges[rangeIndex].name; } void AddReadRequest(const ImageSetIndex& index) override { _baselineData.push(loadData(index)); } void PerformReadRequests(class ProgressListener&) override {} std::unique_ptr GetNextRequested() override { std::unique_ptr data(new BaselineData(_baselineData.top())); _baselineData.pop(); return data; } void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) override; void PerformWriteFlagsTask() override; void PerformWriteDataTask(const ImageSetIndex&, std::vector, std::vector) override { throw std::runtime_error("Not implemented"); } std::string TelescopeName() override { return "Bighorns"; } bool HasCrossCorrelations() const override { return false; } private: struct TimeRange { int start, end; std::string name; TimeRange() {} TimeRange(const TimeRange& source) : start(source.start), end(source.end), name(source.name) {} TimeRange& operator=(const TimeRange& source) { start = source.start; end = source.end; name = source.name; return *this; } }; BHFitsImageSet(const BHFitsImageSet& source); BaselineData loadData(const ImageSetIndex& index); void loadImageData(TimeFrequencyData& data, const TimeFrequencyMetaDataPtr& metaData, const ImageSetIndex& index); std::pair getRangeFromString(const std::string& rangeStr); std::string flagFilePath() const; std::shared_ptr _file; std::stack _baselineData; std::vector _timeRanges; int _width, _height; }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/rfibaselineset.cpp0000664000175000017500000000327214752462134017457 0ustar oleole#include "rfibaselineset.h" #include "../msio/singlebaselinefile.h" #include #include #include "../util/progress/dummyprogresslistener.h" namespace imagesets { RFIBaselineSet::RFIBaselineSet(const std::string& path) : SingleImageSet(), _path(path) {} std::unique_ptr RFIBaselineSet::Read( class ProgressListener& progress) { if (_data.IsEmpty()) { progress.OnStartTask("Reading file"); SingleBaselineFile file; std::ifstream str(_path); file.Read(str, progress); _data = std::move(file.data); _metaData = std::move(file.metaData); _telescopeName = std::move(file.telescopeName); } progress.OnFinish(); return std::unique_ptr(new BaselineData( _data, TimeFrequencyMetaDataPtr(new TimeFrequencyMetaData(_metaData)))); } std::string RFIBaselineSet::BaselineDescription() { if (_metaData.HasAntenna1() && _metaData.HasAntenna2()) { std::ostringstream sstream; sstream << _metaData.Antenna1().station << ' ' << _metaData.Antenna1().name << " x " << _metaData.Antenna2().station << ' ' << _metaData.Antenna2().name; return sstream.str(); } else { return Files().front(); } } void RFIBaselineSet::Write(const std::vector& masks) { SingleBaselineFile file; { std::ifstream str(_path); DummyProgressListener dummy; file.Read(str, dummy); } if (file.data.MaskCount() != masks.size()) throw std::runtime_error( "Number of masks in flag writing action don't match rfibl file"); for (size_t i = 0; i != masks.size(); ++i) file.data.SetMask(i, masks[i]); std::ofstream outstr(_path); file.Write(outstr); } } // namespace imagesets aoflagger-v3.5.1/imagesets/multibandmsimageset.h0000664000175000017500000001240614752462134020162 0ustar oleole#ifndef MULTIBANDMSIMAGESET_H #define MULTIBANDMSIMAGESET_H #include "../msio/baselinereader.h" #include "indexableset.h" #include #include #include #include #include #include namespace imagesets { struct MetaData; /** * The multiband ms image set combines multiple single band measurement * sets by spectrally concatenating the data. * * The \a names contains the measurement sets. They should be part of the * same observation, where every file contains one subband of the * observation. The files are listed in consecutive order and all * subbands should be present. */ class MultiBandMsImageSet final : public IndexableSet { public: MultiBandMsImageSet(const std::vector& names, BaselineIOMode io_mode, const std::string& data_column_name, std::optional start_time_step, std::optional end_time_step, size_t n_threads); MultiBandMsImageSet(const MultiBandMsImageSet&) = delete; ~MultiBandMsImageSet() = default; MultiBandMsImageSet& operator=(const MultiBandMsImageSet&) = delete; void WriteToMs(size_t n_threads); std::unique_ptr Clone() override { throw std::runtime_error("Not available"); } const std::vector& Sequences() const { return sequences_; } size_t Size() const override { return sequences_.size() * BandCount(); } std::string Description(const ImageSetIndex& index) const override; std::string Name() const override { return "Spectrally concatenated set (" + ms_names_.front() + " ...)"; } std::vector Files() const override { return ms_names_; } void AddReadRequest(const ImageSetIndex& index) override; void PerformReadRequests(class ProgressListener& progress) override; std::unique_ptr GetNextRequested() override; void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) override; void PerformWriteFlagsTask() override; void Initialize() override { /* Do nothing.*/ } void PerformWriteDataTask(const ImageSetIndex& index, std::vector realImages, std::vector imaginaryImages) override { throw std::runtime_error("Not implemented"); } BaselineReaderPtr Reader() const override { throw std::runtime_error("Not available"); } std::string TelescopeName() override { casacore::MeasurementSet ms = readers_[0]->OpenMS(); return MSMetaData::GetTelescopeName(ms); } size_t GetAntenna1(const ImageSetIndex& index) const override { return GetSequence(index).antenna1; } size_t GetAntenna2(const ImageSetIndex& index) const override { return GetSequence(index).antenna2; } size_t GetBand(const ImageSetIndex&) const override { return 0; } size_t GetField(const ImageSetIndex& index) const override { return GetSequence(index).fieldId; } size_t GetSequenceId(const ImageSetIndex& index) const override { return GetSequence(index).sequenceId; } std::optional Index(size_t antenna_1, size_t antenna_2, size_t band, size_t sequence_id) const override; size_t BandCount() const override { return 1; } AntennaInfo GetAntennaInfo(unsigned antennaIndex) const override { return antennae_[antennaIndex]; } BandInfo GetBandInfo(unsigned) const override { return band_; } size_t SequenceCount() const override { return observation_times_per_sequence_.size(); } size_t AntennaCount() const override { return antennae_.size(); } FieldInfo GetFieldInfo(unsigned fieldIndex) const override { return fields_[fieldIndex]; } private: size_t GetSequenceIndex(const ImageSetIndex& index) const { return index.Value(); } MSMetaData::Sequence GetSequence(const ImageSetIndex& index) const { return sequences_[GetSequenceIndex(index)]; } size_t EndTimeIndex(size_t sequence_id) const { return observation_times_per_sequence_[sequence_id].size(); } size_t FindBaselineIndex(size_t antenna_1, size_t antenna_2, size_t band, size_t sequence_id) const; void ReadData(size_t n_threads); void ProcessMetaData(); std::unique_ptr CombineData(const ImageSetIndex& index); static const size_t kNotFound; std::vector ms_names_; // TODO Other readers use a vector and call pop_front, if the vector contains // a lot of elements this is inefficient. std::deque> data_; std::vector read_requests_; // All measurement sets read contain the same metadata this a cached copy. std::vector antennae_; std::vector fields_; std::vector sequences_; std::vector> observation_times_per_sequence_; // Contains the information of all concatenated channels BandInfo band_; // Contains the number of channels per band. // This is needed to split the total image per band for the writing step. std::vector channels_per_band_; std::vector> readers_; }; } // namespace imagesets #endif // MULTIBANDMSIMAGESET_H aoflagger-v3.5.1/imagesets/singleimageset.h0000664000175000017500000000452414752462134017126 0ustar oleole#ifndef SINGLEIMAGESET_H #define SINGLEIMAGESET_H #include #include #include #include #include "../structures/types.h" #include "imageset.h" namespace imagesets { class SingleImageSet : public ImageSet { public: SingleImageSet() : ImageSet(), _readCount(0), _lastRead(nullptr), _writeFlagsIndex() {} std::string Name() const override = 0; size_t Size() const override { return 1; } std::string Description(const ImageSetIndex&) const override { return Name(); } std::vector Files() const override = 0; void AddReadRequest(const ImageSetIndex&) override { if (_lastRead != nullptr) { _lastRead.reset(); _readCount = 1; } else { ++_readCount; } } void PerformReadRequests(class ProgressListener& progress) override { _lastRead = Read(progress); _lastRead->SetIndex(StartIndex()); } std::unique_ptr GetNextRequested() override { if (_readCount == 0) throw std::runtime_error("All data reads have already been requested"); if (_lastRead == nullptr) throw std::runtime_error( "GetNextRequested() was called before PerformReadRequests()"); return std::unique_ptr(new BaselineData(*_lastRead)); } virtual std::unique_ptr Read( class ProgressListener& progress) = 0; virtual void Write(const std::vector&) { throw std::runtime_error( "Flag writing is not implemented for this file (SingleImageSet)"); } void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) override { _writeFlagsIndex = index; _writeFlagsMasks = flags; } void PerformWriteFlagsTask() override { if (_writeFlagsIndex.Empty()) throw std::runtime_error("Nothing to write"); Write(_writeFlagsMasks); _writeFlagsIndex = ImageSetIndex(); } virtual std::string BaselineDescription() = 0; bool HasCrossCorrelations() const override { return false; } protected: SingleImageSet(const SingleImageSet&) : _readCount(0), _lastRead(nullptr), _writeFlagsIndex(), _writeFlagsMasks() {} private: int _readCount; std::unique_ptr _lastRead; ImageSetIndex _writeFlagsIndex; std::vector _writeFlagsMasks; }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/fitsimageset.h0000664000175000017500000000777114752462134016621 0ustar oleole#ifndef FITSIMAGESET_H #define FITSIMAGESET_H #include #include #include #include #include #include #include "imageset.h" #include "../structures/antennainfo.h" #include "../structures/types.h" #include "../util/progress/progresslistener.h" namespace imagesets { class FitsImageSet final : public ImageSet { public: explicit FitsImageSet(const std::string& file); ~FitsImageSet(); void Initialize() override; std::unique_ptr Clone() override; size_t Size() const override { return _baselines.size() * _bandCount; } std::string Description(const ImageSetIndex& index) const override; std::string Name() const override { return Files().front(); } std::vector Files() const override; void AddReadRequest(const ImageSetIndex& index) override { _readRequests.emplace_back(index); } void PerformReadRequests(ProgressListener& progress) override { // Immediately clear the _readRequests, to have it empty in case of // exceptions const std::vector requests = std::move(_readRequests); _readRequests.clear(); for (size_t i = 0; i != requests.size(); ++i) { _baselineData.emplace_back(loadData(requests[i])); progress.OnProgress(i + 1, requests.size()); } progress.OnFinish(); } std::unique_ptr GetNextRequested() override { std::unique_ptr data(new BaselineData(_baselineData.back())); _baselineData.resize(_baselineData.size() - 1); return data; } void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) override; void PerformWriteFlagsTask() override; void PerformWriteDataTask(const ImageSetIndex&, std::vector, std::vector) override { throw std::runtime_error("Not implemented"); } std::string TelescopeName() override; bool HasCrossCorrelations() const override { return false; } const std::vector>& Baselines() const { return _baselines; } size_t BandCount() const { return _bandCount; } class AntennaInfo GetAntennaInfo(unsigned antennaIndex) const { return _antennaInfos[antennaIndex]; } const std::string& SourceName() const { return _sourceName; } bool IsDynSpectrumType() const { return _fitsType == DynSpectrumType; } private: FitsImageSet(const FitsImageSet& source); BaselineData loadData(const ImageSetIndex& index); void ReadPrimarySingleTable(TimeFrequencyData& data, TimeFrequencyMetaData& metaData); void ReadTable(TimeFrequencyData& data, TimeFrequencyMetaData& metaData, size_t bandIndex); void ReadAntennaTable(TimeFrequencyMetaData& metaData); void ReadFrequencyTable(TimeFrequencyData& data, TimeFrequencyMetaData& metaData); void ReadCalibrationTable(); void ReadSingleDishTable(TimeFrequencyData& data, TimeFrequencyMetaData& metaData, size_t ifIndex); void ReadDynSpectrum(TimeFrequencyData& data, TimeFrequencyMetaData& metaData); TimeFrequencyData ReadPrimaryGroupTable(size_t baselineIndex, int band, int stokes, TimeFrequencyMetaData& metaData); void saveSingleDishFlags(const std::vector& flags, size_t ifIndex); void saveDynSpectrumFlags(const std::vector& flags); std::shared_ptr _file; std::vector> _baselines; size_t _bandCount; std::vector _antennaInfos; std::map _bandInfos; std::vector _bandIndexToNumber; size_t _currentBaselineIndex, _currentBandIndex; double _frequencyOffset; std::string _sourceName; std::vector _readRequests; std::vector _baselineData; enum { UVFitsType, SDFitsType, DynSpectrumType } _fitsType; }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/pngreader.cpp0000664000175000017500000000432714756152073016433 0ustar oleole#include "pngreader.h" #include #include "../util/progress/progresslistener.h" std::unique_ptr imagesets::PngReader::Read( class ProgressListener& progress) { FILE* fp = fopen(_path.c_str(), "rb"); if (fp == nullptr) throw std::runtime_error("Could not open file"); png_structp png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); if (!png_ptr) throw std::runtime_error("Error creating png struct"); png_infop info_ptr = png_create_info_struct(png_ptr); if (!info_ptr) { png_destroy_read_struct(&png_ptr, nullptr, nullptr); throw std::runtime_error("Error creating png read struct"); } png_init_io(png_ptr, fp); png_read_info(png_ptr, info_ptr); const unsigned width = png_get_image_width(png_ptr, info_ptr); const unsigned height = png_get_image_height(png_ptr, info_ptr); const unsigned color_type = png_get_color_type(png_ptr, info_ptr); const unsigned bit_depth = png_get_bit_depth(png_ptr, info_ptr); Logger::Debug << "Png file: " << width << 'x' << height << " colortype=" << color_type << ", bit_depth=" << bit_depth << '\n'; png_read_update_info(png_ptr, info_ptr); /* read file */ if (setjmp(png_jmpbuf(png_ptr))) throw std::runtime_error("Png error"); png_bytep* row_pointers = (png_bytep*)malloc(sizeof(png_bytep) * height); for (size_t y = 0; y < height; y++) row_pointers[y] = (png_byte*)malloc(png_get_rowbytes(png_ptr, info_ptr)); png_read_image(png_ptr, row_pointers); fclose(fp); const Image2DPtr image = Image2D::CreateZeroImagePtr(width, height); const size_t bytesPerSample = 4; for (size_t f = 0; f < height; ++f) { for (size_t t = 0; t < width; ++t) { int r = row_pointers[f][t * bytesPerSample], g = row_pointers[f][t * bytesPerSample + 1], b = row_pointers[f][t * bytesPerSample + 2]; image->SetValue(t, height - 1 - f, r + g + b); } } const TimeFrequencyData tfData(TimeFrequencyData::AmplitudePart, aocommon::Polarization::StokesI, image); progress.OnFinish(); return std::unique_ptr( new BaselineData(tfData, TimeFrequencyMetaDataCPtr())); } aoflagger-v3.5.1/imagesets/parmimageset.h0000664000175000017500000000376114752462134016606 0ustar oleole#ifndef PARM_IMAGE_H #define PARM_IMAGE_H #include #include #include #include #include #include #include "../structures/types.h" #include "../structures/timefrequencydata.h" #include "../structures/timefrequencymetadata.h" #include "../lua/telescopefile.h" #include "imageset.h" namespace imagesets { class ParmImageSet final : public ImageSet { public: ParmImageSet(const std::string& path) : _path(path), _parmTable(nullptr) {} ~ParmImageSet() override; size_t Size() const override { return AntennaCount(); } inline std::string Description(const ImageSetIndex& index) const override { return AntennaName(index.Value()); } std::unique_ptr Clone() override { throw std::runtime_error("Cannot copy set"); } void Initialize() override; std::string Name() const override { return "Parmdb"; } std::vector Files() const override { return std::vector{_path}; } std::string TelescopeName() override { return TelescopeFile::TelescopeName(TelescopeFile::GENERIC_TELESCOPE); } TimeFrequencyData* LoadData(const ImageSetIndex& index); void AddReadRequest(const ImageSetIndex& index) override { TimeFrequencyData* data = LoadData(index); BaselineData* baseline = new BaselineData(*data, TimeFrequencyMetaDataCPtr(), index); delete data; _baselineBuffer.push_back(baseline); } void PerformReadRequests(class ProgressListener&) override {} std::unique_ptr GetNextRequested() override { std::unique_ptr baseline(std::move(_baselineBuffer.front())); _baselineBuffer.pop_front(); return baseline; } unsigned AntennaCount() const { return _antennas.size(); } std::string AntennaName(unsigned index) const { return _antennas[index]; } private: const std::string _path; std::vector _antennas; class ParmTable* _parmTable; std::deque _baselineBuffer; }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/msoptions.h0000664000175000017500000000057714752462134016165 0ustar oleole#ifndef MS_OPTIONS_H #define MS_OPTIONS_H #include #include #include "../structures/types.h" #include "../aoluarunner/options.h" struct MSOptions { BaselineIOMode ioMode; std::string dataColumnName; bool combineSPWs; bool concatenateFrequency; std::optional intervalStart, intervalEnd; BaselineIntegration baselineIntegration; }; #endif aoflagger-v3.5.1/imagesets/multibandmsimageset.cpp0000664000175000017500000004537014752462134020523 0ustar oleole #include "multibandmsimageset.h" #include "../msio/directbaselinereader.h" #include "../msio/memorybaselinereader.h" #include "../msio/reorderingbaselinereader.h" #include "../util/logger.h" #include "../util/progress/dummyprogresslistener.h" #include "../util/progress/subtasklistener.h" #include "../util/stopwatch.h" #include "joinedspwset.h" #include #include #include #include #include namespace imagesets { const size_t MultiBandMsImageSet::kNotFound = std::numeric_limits::max(); static std::unique_ptr CreateReader(const std::string& ms_name, BaselineIOMode io_mode) { switch (io_mode) { case BaselineIOMode::DirectReadMode: return std::make_unique(ms_name); case BaselineIOMode::ReorderingReadMode: return std::make_unique(ms_name); case BaselineIOMode::AutoReadMode: case BaselineIOMode::MemoryReadMode: return std::make_unique(ms_name); } assert(false); return nullptr; } MultiBandMsImageSet::MultiBandMsImageSet( const std::vector& ms_names, BaselineIOMode io_mode, const std::string& data_column_name, std::optional start_time_step, std::optional end_time_step, size_t n_threads) : ms_names_(ms_names) { // AutoReadMode behaves as-if MemoryReadMode. When the estimated amount of // memory is insufficent switch to the direct reader. This behaviour matches // MSImageSet::initReader. if (io_mode == BaselineIOMode::AutoReadMode && !MemoryBaselineReader::IsEnoughMemoryAvailable( ms_names.size() * BaselineReader::MeasurementSetIntervalDataSize( ms_names[0], start_time_step, end_time_step))) io_mode = BaselineIOMode::ReorderingReadMode; for (const std::string& ms_name : ms_names_) { std::unique_ptr& reader = readers_.emplace_back(CreateReader(ms_name, io_mode)); reader->SetInterval(start_time_step, end_time_step); reader->SetDataColumnName(data_column_name); } assert(n_threads != 0 && n_threads <= readers_.size() && "Caller should provide a valid number of execution threads."); ReadData(n_threads); ProcessMetaData(); } static std::vector GetWriters( std::vector>& readers) { std::vector writers; // Only the modified readers need to be written. for (std::unique_ptr& reader : readers) if (reader->IsModified()) writers.emplace_back(reader.get()); return writers; } void MultiBandMsImageSet::WriteToMs(size_t n_threads) { assert(n_threads != 0 && n_threads <= readers_.size() && "Caller should provide a valid number of execution threads."); const Stopwatch watch(true); const std::vector writers = GetWriters(readers_); aocommon::ThreadPool::GetInstance().SetNThreads(n_threads); aocommon::DynamicFor executor; executor.Run(0, writers.size(), [&](size_t i) { writers[i]->WriteToMs(); }); Logger::Debug << "Writing took " << watch.ToString() << ".\n"; } std::optional MultiBandMsImageSet::Index( size_t antenna_1, size_t antenna_2, size_t band, size_t sequence_id) const { const size_t value = FindBaselineIndex(antenna_1, antenna_2, band, sequence_id); if (value != kNotFound) { return ImageSetIndex(Size(), value); } return {}; } static std::pair> CombineBands( const std::vector& meta_data) { assert(meta_data[0]->BandCount() == 1 && "The reader should have validated the number of bands"); BandInfo band_info = meta_data[0]->GetBandInfo(0); std::vector& channels = band_info.channels; std::vector channels_per_band; channels_per_band.push_back(channels.size()); std::for_each( meta_data.begin() + 1, meta_data.end(), [&](const MSMetaData* element) { assert(element->BandCount() == 1 && "The reader should have validated the number of bands"); const BandInfo& band = element->GetBandInfo(0); std::copy(band.channels.begin(), band.channels.end(), std::back_inserter(channels)); channels_per_band.push_back(band.channels.size()); }); return {std::move(band_info), std::move(channels_per_band)}; } static const std::vector>& GetBaselines( const MSMetaData* meta_data) { return meta_data->GetBaselines(); } static const std::set& GetObservationTimes( const MSMetaData* meta_data) { return meta_data->GetObservationTimes(); } static const std::vector>& GetObservationTimesPerSequence( const MSMetaData* meta_data) { return meta_data->GetObservationTimesPerSequence(); } static const std::vector& GetAntennae( const MSMetaData* meta_data) { return meta_data->GetAntennas(); } static const std::vector& GetFields(const MSMetaData* meta_data) { return meta_data->GetFields(); } static const std::vector& GetSequences( const MSMetaData* meta_data) { return meta_data->GetSequences(); } template static void ValidateEqual(const Type& lhs, std::vector::const_iterator first, std::vector::const_iterator last, Functor&& functor, const std::string& description) { if (!std::all_of(first, last, [&](const MSMetaData* element) { return lhs == functor(element); })) { throw std::runtime_error( "The loaded measurement sets are not compatible in this dimension: " + description); } } template static auto ExtractField(const std::vector& meta_data, Functor&& functor, const std::string& description) { assert(!meta_data.empty()); auto result = functor(meta_data[0]); ValidateEqual(result, meta_data.begin() + 1, meta_data.end(), std::forward(functor), description); return result; } void MultiBandMsImageSet::ReadData(size_t n_threads) { const Stopwatch watch(true); aocommon::ThreadPool::GetInstance().SetNThreads(n_threads); aocommon::DynamicFor executor; executor.Run(0, readers_.size(), [&](size_t i) { readers_[i]->PrepareReadWrite(BaselineReader::dummy_progress_); }); Logger::Debug << "Reading took " << watch.ToString() << ".\n"; } // Returns the metadata of the readers and initializes their main tables. static std::vector GetInitializedMetaData( std::vector>::iterator first, std::vector>::iterator last) { std::vector result; std::transform(first, last, std::back_inserter(result), [](std::unique_ptr& reader) { MSMetaData& meta_data{reader->MetaData()}; meta_data.InitializeMainTableData(); return &meta_data; }); return result; } namespace { // These two helper functions below are based on // https://stackoverflow.com/questions/17074324/how-can-i-sort-two-vectors-in-the-same-way-with-criteria-that-uses-only-one-of // They are used in MultiBandMsImageSet::ProcessMetaData() to sort multiple // vectors simulatenously by frequency. There is a nicer ranges based solution, // supported from C++23 onwards, see the comment where the functions are used. // The functions here can be removed when moving to to ranges based solution /** * Returns an indexing vector that sorts vector \p input according to comparator * \p compare */ template std::vector MakeSortingPermutation(const std::vector& input, const Compare& compare) { std::vector permutation(input.size()); std::iota(permutation.begin(), permutation.end(), 0); std::sort(permutation.begin(), permutation.end(), [&](std::size_t i, std::size_t j) { return compare(input[i], input[j]); }); return permutation; } /** * Permute the vector \p input using indexing vector \p permutation. */ template void ApplyPermutation(std::vector& input, const std::vector& permutation) { std::vector done(input.size()); for (std::size_t i = 0; i < input.size(); ++i) { if (done[i]) { continue; } done[i] = true; std::size_t prev_j = i; std::size_t j = permutation[i]; while (i != j) { std::swap(input[prev_j], input[j]); done[j] = true; prev_j = j; j = permutation[j]; } } } /** * Find the order in which the channels in the spws are sorted. * * Returns std::optional(true) when all spws are in ascending frequency * order, std::optiona(false) when in descending frequency order. * std::optional() when in mixed order */ std::optional IsInAscendingFrequencyOrder( const std::vector& meta_data) { std::optional frequency_is_ascending; for (const MSMetaData* ms_meta_data : meta_data) { // Only check the first band, assuming the rest is in the same order const std::vector& channels = ms_meta_data->GetBandInfo(0).channels; // If there is only one channel, there is no ordering if (channels.size() == 1) continue; // Determine the ordering of this ms from the order of the first twp // channels bool ms_frequency_is_ascending = channels[0].frequencyHz < channels[1].frequencyHz; if (frequency_is_ascending.has_value()) { // If an ordering has already been established, the current ms should be // conformal if (*frequency_is_ascending != ms_frequency_is_ascending) { return std::optional(); } } else { // There was no ordering established, the current ms determines the // ordering frequency_is_ascending = ms_frequency_is_ascending; } } // No ordering has been established (all ms have a single channel) // Set the ordering arbitrarily to ascending if (!frequency_is_ascending.has_value()) frequency_is_ascending = true; return frequency_is_ascending; } } // anonymous namespace void MultiBandMsImageSet::ProcessMetaData() { std::vector meta_data = GetInitializedMetaData(readers_.begin(), readers_.end()); // The ms's should be combined in frequency order // Either ascending or descending depending on the order of the channels in // the ms std::optional frequency_is_ascending = IsInAscendingFrequencyOrder(meta_data); if (!frequency_is_ascending.has_value()) { throw std::runtime_error( "Trying to concatenate MeasurementSets that have not the same " "ordering in frequency."); } // In C++23 multiple vectors can be sorted simultaneously using ranges: // // std::ranges::sort(std::views::zip(ms_names_, readers_, meta_data), // [frequency_is_ascending](auto&& a, auto&& b) { // return (std::get<2>(a)->GetBandInfo(0).channels[0].frequencyHz < // std::get<2>(b)->GetBandInfo(0).channels[0].frequencyHz) != // !(*frequency_is_ascending); // }); // // Without ranges support, the helper functions MakeSortingPermutation and // ApplyPermutation are needed // Get the permutation to sort the ms's in ascending/descending frequency // order The != operator is equivalent to a xor operation, toggling the // comparison std::vector permutation = MakeSortingPermutation( meta_data, [frequency_is_ascending](const MSMetaData* a, const MSMetaData* b) { return (a->GetBandInfo(0).channels[0].frequencyHz < b->GetBandInfo(0).channels[0].frequencyHz) != !(*frequency_is_ascending); }); // Apply the permutation to ms_names_ and its derivatives ApplyPermutation(ms_names_, permutation); ApplyPermutation(readers_, permutation); ApplyPermutation(meta_data, permutation); // These fields are only validated. ExtractField(meta_data, GetBaselines, "baselines"); ExtractField(meta_data, GetObservationTimes, "timesteps"); // These fields are validated and cached. antennae_ = ExtractField(meta_data, GetAntennae, "antennas"); fields_ = ExtractField(meta_data, GetFields, "fields"); sequences_ = ExtractField(meta_data, GetSequences, "sequences"); observation_times_per_sequence_ = ExtractField( meta_data, GetObservationTimesPerSequence, "timesteps per sequence"); std::tie(band_, channels_per_band_) = CombineBands(meta_data); } size_t MultiBandMsImageSet::FindBaselineIndex(size_t antenna_1, size_t antenna_2, size_t /* band */, size_t sequence_id) const { // TODO This is a linear search, when it becomes measureable it should be // improved. size_t index = 0; for (const MSMetaData::Sequence& sequence : sequences_) { const bool antennaMatch = (sequence.antenna1 == antenna_1 && sequence.antenna2 == antenna_2) || (sequence.antenna1 == antenna_2 && sequence.antenna2 == antenna_1); if (antennaMatch && sequence.sequenceId == sequence_id) { return index; } ++index; } return kNotFound; } std::string MultiBandMsImageSet::Description(const ImageSetIndex& index) const { std::stringstream sstream; const MSMetaData::Sequence& sequence = sequences_[GetSequenceIndex(index)]; const AntennaInfo& antenna_1 = antennae_[sequence.antenna1]; const AntennaInfo& antenna_2 = antennae_[sequence.antenna2]; sstream << antenna_1.station << ' ' << antenna_1.name << " x " << antenna_2.station << ' ' << antenna_2.name; const double band_start = round(band_.channels.front().frequencyHz / 100000.0) / 10.0; const double band_end = round(band_.channels.back().frequencyHz / 100000.0) / 10.0; sstream << ", spectrally concatenated (" << band_start << "MHz -" << band_end << "MHz)"; if (SequenceCount() > 1) { sstream << ", seq " << sequence.sequenceId; } return sstream.str(); } static TimeFrequencyMetaDataCPtr GetMetaData( BaselineReader& reader, const MSMetaData::Sequence& sequence, const std::vector& uvw) { auto result = std::make_unique(); MSMetaData& meta_data = reader.MetaData(); result->SetAntenna1(meta_data.GetAntennaInfo(sequence.antenna1)); result->SetAntenna2(meta_data.GetAntennaInfo(sequence.antenna2)); result->SetBand(meta_data.GetBandInfo(0)); result->SetField(meta_data.GetFieldInfo(sequence.fieldId)); const std::set& observation_times = meta_data.GetObservationTimesSet(sequence.sequenceId); result->SetObservationTimes( std::vector(observation_times.begin(), observation_times.end())); result->SetUVW(uvw); return TimeFrequencyMetaDataCPtr{result.release()}; } static std::unique_ptr GetData( BaselineReader& reader, const MSMetaData::Sequence& sequence, const imagesets::ImageSetIndex& index) { std::vector uvw; const TimeFrequencyData data = reader.GetNextResult(uvw); const TimeFrequencyMetaDataCPtr meta_data = GetMetaData(reader, sequence, uvw); return std::make_unique(std::move(data), std::move(meta_data), index); } std::unique_ptr MultiBandMsImageSet::CombineData( const ImageSetIndex& index) { const MSMetaData::Sequence& sequence = GetSequence(index); std::vector> data; size_t height = 0; for (std::unique_ptr& reader : readers_) { data.emplace_back(GetData(*reader, sequence, index)); height += data.back()->Data().ImageHeight(); } return JoinedSPWSet::CombineBaselineData(std::move(data), height, index); } void MultiBandMsImageSet::AddReadRequest(const ImageSetIndex& index) { const size_t kStartTimeIndex = 0; const MSMetaData::Sequence& sequence = sequences_[GetSequenceIndex(index)]; const size_t end_time_index = EndTimeIndex(sequence.sequenceId); for (std::unique_ptr& reader : readers_) { reader->AddReadRequest(sequence.antenna1, sequence.antenna2, sequence.spw, sequence.sequenceId, kStartTimeIndex, end_time_index); } read_requests_.emplace_back(index); } void MultiBandMsImageSet::PerformReadRequests(ProgressListener& progress) { if (!data_.empty()) { throw std::runtime_error( "PerformReadRequest() called, but a previous read request was not " "completely processed."); } for (size_t i = 0; i != readers_.size(); ++i) { SubTaskListener listener(progress, i, readers_.size()); readers_[i]->PerformReadRequests(listener); } progress.OnFinish(); for (const ImageSetIndex& index : read_requests_) { data_.emplace_back(CombineData(index)); } read_requests_.clear(); } std::unique_ptr MultiBandMsImageSet::GetNextRequested() { std::unique_ptr result = std::move(data_.front()); data_.pop_front(); if (result->Data().IsEmpty()) { throw std::runtime_error( "Calling GetNextRequested(), but requests were not read with " "LoadRequests."); } return result; } } // namespace imagesets namespace imagesets { void MultiBandMsImageSet::AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) { const size_t n_polarizations = readers_[0]->Polarizations().size(); if (flags.size() > n_polarizations) { throw std::runtime_error( "Trying to write more polarizations to image set than available"); } std::vector all_flags; if (flags.size() < n_polarizations) { if (flags.size() != 1) { throw std::runtime_error( "Incorrect number of polarizations in write action"); } all_flags.resize(n_polarizations, flags[0]); } else { all_flags = flags; } size_t offset = 0; const MSMetaData::Sequence& sequence = GetSequence(index); for (size_t i = 0; i < readers_.size(); ++i) { std::vector reader_flags; const size_t count = channels_per_band_[i]; for (size_t j = 0; j < all_flags.size(); ++j) reader_flags.emplace_back( Mask2D::CreatePtrFromRows(*all_flags[j], offset, count)); offset += count; readers_[i]->AddWriteTask(reader_flags, sequence.antenna1, sequence.antenna2, 0, sequence.sequenceId); } } void MultiBandMsImageSet::PerformWriteFlagsTask() { for (std::unique_ptr& reader : readers_) reader->PerformFlagWriteRequests(); } } // namespace imagesets aoflagger-v3.5.1/imagesets/qualitystatimageset.h0000664000175000017500000001137414756152073020234 0ustar oleole#ifndef QUALITY_STAT_IMAGE_SET_H #define QUALITY_STAT_IMAGE_SET_H #include "singleimageset.h" #include #include #include #include #include "../quality/qualitytablesformatter.h" #include "../quality/statisticsderivator.h" #include "../util/logger.h" #include "../lua/telescopefile.h" #include "../util/progress/progresslistener.h" #include #include #include namespace imagesets { class QualityStatImageSet final : public SingleImageSet { public: explicit QualityStatImageSet(const std::string& filename) : _filename(filename), _statisticKind(QualityTablesFormatter::StandardDeviationStatistic) { if (!_filename.empty() && (*_filename.rbegin()) == '/') _filename.resize(_filename.size() - 1); std::filesystem::path p(_filename); if (p.filename() == "QUALITY_TIME_STATISTIC") _filename = p.parent_path().string(); } std::string Name() const override { return _filename; } std::vector Files() const override { return std::vector{_filename}; } std::string BaselineDescription() override { return Name(); } std::string TelescopeName() override { return TelescopeFile::TelescopeName(TelescopeFile::GENERIC_TELESCOPE); } std::unique_ptr Read(ProgressListener& progress) override { QualityTablesFormatter formatter(_filename); StatisticsCollection statCollection; statCollection.Load(formatter); StatisticsDerivator derivator(statCollection); std::pair data = derivator.CreateTFData(_statisticKind); TimeFrequencyData& tfData = data.first; TimeFrequencyMetaDataCPtr& metaData = data.second; Mask2DPtr mask = Mask2D::CreateUnsetMaskPtr(tfData.ImageWidth(), tfData.ImageHeight()); for (size_t y = 0; y != tfData.ImageHeight(); ++y) { for (size_t x = 0; x != tfData.ImageWidth(); ++x) { for (size_t i = 0; i != tfData.ImageCount(); ++i) mask->SetValue(x, y, !std::isfinite(tfData.GetImage(i)->Value(x, y))); } } tfData.SetGlobalMask(mask); progress.OnFinish(); return std::unique_ptr(new BaselineData(tfData, metaData)); } std::unique_ptr Clone() override { return std::unique_ptr(new QualityStatImageSet(_filename)); } void Initialize() override {} void Write(const std::vector& flags) override { std::vector flagsCopy(flags); casacore::MeasurementSet ms(_filename, casacore::Table::Update); if (ms.nrow() > 0) { casacore::ArrayColumn flagColumn( ms, casacore::MeasurementSet::columnName(casacore::MSMainEnums::FLAG)); casacore::ROScalarColumn timeColumn( ms, casacore::MeasurementSet::columnName(casacore::MSMainEnums::TIME)); casacore::Array flagArray(flagColumn.get(0)); const size_t polCount = flagArray.shape()[0], channelCount = flagArray.shape()[1]; Logger::Debug << "Saving flags to measurement set (" << channelCount << " ch x " << polCount << " pol)...\n"; if (flagsCopy.size() == 1 && polCount > 1) { do { flagsCopy.push_back(flagsCopy[0]); } while (flagsCopy.size() != polCount); } if (flagsCopy.size() != polCount) throw std::runtime_error("Polarization counts don't match"); if (flagsCopy[0]->Height() != channelCount) { std::ostringstream s; s << "Channel counts don't match (" << flagsCopy[0]->Height() << " in mask, " << channelCount << " in data)"; throw std::runtime_error(s.str()); } size_t timeIndex = size_t(-1); double time = -1.0; std::unique_ptr timestepFlags(new bool[channelCount * polCount]); for (size_t row = 0; row != ms.nrow(); ++row) { if (time != timeColumn(row)) { time = timeColumn(row); timeIndex++; bool* iter = timestepFlags.get(); for (size_t ch = 0; ch != channelCount; ++ch) { for (size_t p = 0; p != polCount; ++p) { *iter = flagsCopy[p]->Value(timeIndex, ch); ++iter; } } } flagColumn.get(row, flagArray); casacore::Array::contiter iter = flagArray.cbegin(); for (size_t i = 0; i != channelCount * polCount; ++i) { *iter = *iter || timestepFlags[i]; ++iter; } flagColumn.put(row, flagArray); } timestepFlags.reset(); } } private: std::string _filename; QualityTablesFormatter::StatisticKind _statisticKind; }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/imageset.h0000664000175000017500000001260614756365521015732 0ustar oleole#ifndef GUI_IMAGESET_H #define GUI_IMAGESET_H #include #include #include #include #include "imagesetindex.h" #include "../structures/types.h" #include "../structures/timefrequencydata.h" #include "../structures/timefrequencymetadata.h" class ProgressListener; struct MSOptions; namespace imagesets { class BaselineData { public: BaselineData(const TimeFrequencyData& data, const TimeFrequencyMetaDataCPtr& metaData, const ImageSetIndex& index) : _data(data), _metaData(metaData), _index(index) {} explicit BaselineData(const ImageSetIndex& index) : _data(), _metaData(), _index(index) {} BaselineData(const TimeFrequencyData& data, const TimeFrequencyMetaDataCPtr& metaData) : _data(data), _metaData(metaData), _index() {} explicit BaselineData(const TimeFrequencyMetaDataCPtr& metaData) : _data(), _metaData(metaData), _index() {} BaselineData() : _data(), _metaData(), _index() {} BaselineData(const BaselineData& source) = default; BaselineData(BaselineData&& source) = default; BaselineData& operator=(const BaselineData& source) = default; BaselineData& operator=(BaselineData&& source) = default; const TimeFrequencyData& Data() const { return _data; } void SetData(const TimeFrequencyData& data) { _data = data; } TimeFrequencyMetaDataCPtr MetaData() const { return _metaData; } void SetMetaData(TimeFrequencyMetaDataCPtr metaData) { _metaData = metaData; } const ImageSetIndex& Index() const { return _index; } ImageSetIndex& Index() { return _index; } void SetIndex(const ImageSetIndex& newIndex) { _index = newIndex; } private: TimeFrequencyData _data; TimeFrequencyMetaDataCPtr _metaData; ImageSetIndex _index; }; class ImageSet { public: virtual ~ImageSet() {} virtual std::unique_ptr Clone() = 0; virtual std::string Description(const ImageSetIndex& index) const = 0; virtual size_t Size() const = 0; /** * Initialize is used to initialize the image set after it has been created * and after all possible options have been set that might influence * initialization (such as number of parts to read). */ virtual void Initialize() = 0; virtual std::string Name() const = 0; virtual std::vector Files() const = 0; virtual std::string TelescopeName() = 0; virtual void AddReadRequest(const ImageSetIndex& index) = 0; virtual void PerformReadRequests(class ProgressListener& progress) = 0; virtual std::unique_ptr GetNextRequested() = 0; virtual void AddWriteFlagsTask(const ImageSetIndex& /*index*/, std::vector& /*flags*/) { throw std::runtime_error( "Adding a flag writing task is not implemented for this file format"); } virtual void PerformWriteFlagsTask() { throw std::runtime_error( "Performing a flag writing task is not implemented for this file " "format"); } virtual void PerformWriteDataTask( const ImageSetIndex& /*index*/, std::vector /*_realImages*/, std::vector /*_imaginaryImages*/) { throw std::runtime_error( "Performing a data writing task is not implemented for this file " "format"); } /** * If an imageset has the concept of cross correlations, this returns true. * When true, aoflagger will by default only process baselines that are formed * from two different antennas. If false, it will process all baselines by * default, which is appropriate for single-dish image sets. */ virtual bool HasCrossCorrelations() const { return true; } static std::unique_ptr Create(const std::vector& files, const MSOptions& options); static bool IsSdhdfFile(const std::string& file); static bool IsFitsFile(const std::string& file); static bool IsBHFitsFile(const std::string& file); static bool IsH5File(const std::string& file); static bool IsRCPRawFile(const std::string& file); static bool IsTKPRawFile(const std::string& file); static bool IsRawDescFile(const std::string& file); static bool IsParmFile(const std::string& file); static bool IsTimeFrequencyStatFile(const std::string& file); static bool IsMSFile(const std::string& file); static bool IsNoiseStatFile(const std::string& file); static bool IsPngFile(const std::string& file); static bool IsFilterBankFile(const std::string& file); static bool IsQualityStatSet(const std::string& file); static bool IsRFIBaselineSet(const std::string& file); ImageSetIndex StartIndex() const { return ImageSetIndex(Size()); } void AddWriteFlagsTask(const ImageSetIndex& index, const TimeFrequencyData& data) { std::vector flags; for (size_t i = 0; i != data.MaskCount(); ++i) flags.push_back(data.GetMask(i)); AddWriteFlagsTask(index, flags); } void PerformWriteDataTask(const ImageSetIndex& index, const TimeFrequencyData& data) { std::vector realImages, imaginaryImages; for (size_t i = 0; i != data.PolarizationCount(); ++i) { TimeFrequencyData polData(data.MakeFromPolarizationIndex(i)); realImages.push_back(polData.GetRealPart()); imaginaryImages.push_back(polData.GetImaginaryPart()); } PerformWriteDataTask(index, realImages, imaginaryImages); } }; } // namespace imagesets #endif aoflagger-v3.5.1/imagesets/imageset.cpp0000664000175000017500000001261515065216451016254 0ustar oleole#include "imageset.h" #include "averagingmsset.h" #include "bhfitsimageset.h" #include "coaddedimageset.h" #include "filterbankset.h" #include "fitsimageset.h" #include "h5imageset.h" #include "msimageset.h" #include "msoptions.h" #include "parmimageset.h" #include "pngreader.h" #include "qualitystatimageset.h" #include "rfibaselineset.h" #include "sdhdfimageset.h" #include namespace imagesets { std::unique_ptr ImageSet::Create( const std::vector& files, const MSOptions& options) { using P = std::unique_ptr; if (files.size() == 1) { const std::string& file = files.front(); if (IsFitsFile(file)) { return P(new FitsImageSet(file)); } else if (IsBHFitsFile(file)) { return P(new BHFitsImageSet(file)); } else if (IsH5File(file)) { return P(new H5ImageSet(file)); } else if (IsRCPRawFile(file)) { throw std::runtime_error("Don't know how to open RCP raw files"); } else if (IsTKPRawFile(file)) { throw std::runtime_error("Don't know how to open TKP raw files"); } else if (IsRawDescFile(file)) { throw std::runtime_error("Don't know how to open RCP desc files"); } else if (IsParmFile(file)) { return P(new ParmImageSet(file)); } else if (IsPngFile(file)) { return P(new PngReader(file)); } else if (IsFilterBankFile(file)) { return P(new FilterBankSet(file)); } else if (IsQualityStatSet(file)) { return P(new QualityStatImageSet(file)); } else if (IsRFIBaselineSet(file)) { return P(new RFIBaselineSet(file)); } else if (IsSdhdfFile(file)) { return P(new SdhdfImageSet(file)); } else { // it's an MS if (options.baselineIntegration.enable.value_or(false)) return P(new AveragingMsSet( file, options.dataColumnName, options.baselineIntegration.mode.value_or( BaselineIntegrationMode::Average), options.baselineIntegration.differencing.value_or( BaselineIntegrationDifferencing::NoDifference), options.baselineIntegration.withAutos.value_or(false), options.baselineIntegration.withFlagged.value_or(false))); else return P(new MSImageSet(file, options.ioMode)); } } else { return P(new CoaddedImageSet(files, options.ioMode)); } } bool ImageSet::IsH5File(const std::string& file) { return (file.size() > 3 && boost::to_upper_copy(file.substr(file.size() - 3)) == ".H5"); } bool ImageSet::IsBHFitsFile(const std::string& file) { const std::string uppFile(boost::to_upper_copy(file)); return (uppFile.size() > 7 && uppFile.substr(file.size() - 7) == ".BHFITS"); } bool ImageSet::IsFitsFile(const std::string& file) { const std::string uppFile(boost::to_upper_copy(file)); return (uppFile.size() > 4 && uppFile.substr(file.size() - 4) == ".UVF") || (uppFile.size() > 5 && uppFile.substr(file.size() - 5) == ".FITS") || (uppFile.size() > 7 && uppFile.substr(file.size() - 7) == ".UVFITS") || (uppFile.size() > 7 && uppFile.substr(file.size() - 7) == ".SDFITS"); // Parkes raw files are named like this } bool ImageSet::IsRCPRawFile(const std::string& file) { return file.size() > 4 && file.substr(file.size() - 4) == ".raw"; } bool ImageSet::IsTKPRawFile(const std::string& file) { return file.size() > 4 && file.substr(file.size() - 4) == ".1ch"; } bool ImageSet::IsRawDescFile(const std::string& file) { return file.size() > 8 && file.substr(file.size() - 8) == ".rawdesc"; } bool ImageSet::IsParmFile(const std::string& file) { return file.size() >= 10 && file.substr(file.size() - 10) == "instrument"; } bool ImageSet::IsTimeFrequencyStatFile(const std::string& file) { return (file.size() >= 24 && file.substr(file.size() - 24) == "counts-timefreq-auto.txt") || (file.size() >= 25 && file.substr(file.size() - 25) == "counts-timefreq-cross.txt"); } bool ImageSet::IsNoiseStatFile(const std::string& file) { return file.find("noise-statistics-tf") != std::string::npos && file.find("txt") != std::string::npos; } bool ImageSet::IsPngFile(const std::string& file) { return file.size() >= 4 && file.substr(file.size() - 4) == ".png"; } bool ImageSet::IsFilterBankFile(const std::string& file) { return file.size() >= 4 && file.substr(file.size() - 4) == ".fil"; } bool ImageSet::IsQualityStatSet(const string& file) { if (file.empty()) return false; std::string copy(file); if (*copy.rbegin() == '/') copy.resize(copy.size() - 1); std::filesystem::path p(copy); return p.filename() == "QUALITY_TIME_STATISTIC"; } bool ImageSet::IsRFIBaselineSet(const std::string& file) { return file.size() >= 6 && file.substr(file.size() - 6) == ".rfibl"; } bool ImageSet::IsSdhdfFile(const std::string& file) { return (file.size() >= 4 && file.substr(file.size() - 4) == ".hdf") || (file.size() >= 6 && file.substr(file.size() - 6) == ".sdhdf"); } bool ImageSet::IsMSFile(const std::string& file) { return (!IsBHFitsFile(file)) && (!IsFitsFile(file)) && (!IsRCPRawFile(file)) && (!IsTKPRawFile(file)) && (!IsRawDescFile(file)) && (!IsParmFile(file)) && (!IsTimeFrequencyStatFile(file)) && (!IsNoiseStatFile(file)) && (!IsPngFile(file)) && (!IsFilterBankFile(file)) && (!IsQualityStatSet(file)) && (!IsRFIBaselineSet(file)) && (!IsSdhdfFile(file)); } } // namespace imagesets aoflagger-v3.5.1/imagesets/filterbankset.h0000664000175000017500000000575615136222016016761 0ustar oleole#ifndef FILTERBANKSET_H #define FILTERBANKSET_H #include #include #include #include #include #include "imageset.h" #include "../util/logger.h" namespace imagesets { /** * Reader for the Sigproc filter bank format. * See https://sigproc.sourceforge.net/sigproc.pdf , chapter 3. */ class FilterBankSet final : public ImageSet { public: explicit FilterBankSet(const std::string& location); FilterBankSet(const FilterBankSet& source); ~FilterBankSet() noexcept = default; std::unique_ptr Clone() override { std::unique_ptr set(new FilterBankSet(*this)); return set; } size_t Size() const override { return interval_count_; } std::string Name() const override { return location_; } std::string Description(const ImageSetIndex& index) const override; std::vector Files() const override { return std::vector{location_}; } std::string TelescopeName() override; void AddReadRequest(const ImageSetIndex& index) override; void PerformReadRequests(class ProgressListener& progress) override; std::unique_ptr GetNextRequested() override; void AddWriteFlagsTask(const ImageSetIndex& index, std::vector& flags) override; void Initialize() override; void PerformWriteDataTask(const ImageSetIndex& index, std::vector realImages, std::vector imaginaryImages) override; void PerformWriteFlagsTask() override {} bool HasCrossCorrelations() const override { return false; } double CentreFrequency() const { return (bank1_centre_frequency + (bank_channel_bandwidth_ * channel_count_ * 0.5)) * 1e6; } double ChannelWidth() const { return std::fabs(bank_channel_bandwidth_) * 1e6; } double TimeResolution() const { return time_of_sample_; } private: friend class FilterBankSetIndex; std::string location_; double time_of_sample_; double start_time_; double bank1_centre_frequency; double bank_channel_bandwidth_; size_t channel_count_; size_t if_count_; size_t bit_count_; size_t sample_count_; size_t n_beams_; size_t i_beam_; int machine_id_; int telescope_id_; size_t interval_count_; std::streampos header_end_; std::deque requests_; std::queue> baselines_; static int32_t ReadInt(std::istream& str) { int32_t val; str.read(reinterpret_cast(&val), sizeof(int32_t)); return val; } static double ReadDouble(std::istream& str) { double val; str.read(reinterpret_cast(&val), sizeof(double)); return val; } static std::string ReadString(std::istream& str) { const int32_t length = ReadInt(str); if (length <= 0 || length >= 80) return std::string(); std::string data(length, 0); str.read(&data[0], length); return std::string(&data[0]); } }; } // namespace imagesets #endif aoflagger-v3.5.1/aoqplot/0000775000175000017500000000000015146315735013450 5ustar oleoleaoflagger-v3.5.1/aoqplot/plotsheet.h0000664000175000017500000000065715105046727015635 0ustar oleole#ifndef PLOT_SHEET_H #define PLOT_SHEET_H #include "../structures/antennainfo.h" #include #include #include class PlotSheet : public Gtk::Box { public: sigc::signal SignalStatusChange() { return _signalStatusChange; } virtual void InitializeToolbar(Gtk::Box& toolbar) {} protected: sigc::signal _signalStatusChange; }; #endif aoflagger-v3.5.1/aoqplot/timefrequencyplotpage.cpp0000664000175000017500000000160714752462134020572 0ustar oleole#include "timefrequencyplotpage.h" #include "controllers/tfpagecontroller.h" TimeFrequencyPlotPage::TimeFrequencyPlotPage(TFPageController* controller) : GrayScalePlotPage(controller) { grayScaleWidget().OnMouseMovedEvent().connect( sigc::mem_fun(*this, &TimeFrequencyPlotPage::onMouseMoved)); } void TimeFrequencyPlotPage::onMouseMoved(double x, double y) { std::stringstream text; const QualityTablesFormatter::StatisticKind kind = getSelectedStatisticKind(); const std::string& kindName = QualityTablesFormatter::KindToName(kind); const MaskedHeatMap& map = static_cast(grayScaleWidget().Plot()); size_t image_x; size_t image_y; if (map.UnitToImage(x, y, image_x, image_y)) { text << kindName << " = " << map.GetImage2D()->Value(image_x, image_y) << " (" << image_x << ", " << image_y << ")"; } _signalStatusChange(text.str()); } aoflagger-v3.5.1/aoqplot/openoptionswindow.h0000664000175000017500000000510515105043202017405 0ustar oleole#ifndef GUI_QUALITY__OPEN_OPTIONS_WINDOW_H #define GUI_QUALITY__OPEN_OPTIONS_WINDOW_H #include #include #include #include #include #include #include class OpenOptionsWindow : public Gtk::Window { public: OpenOptionsWindow() : _downsampleTimeButton("Lower time resolution (faster plots)"), _downsampleFreqButton("Lower frequency resolution (faster plots)"), _correctHistograms("Correct histograms for frequence response"), _cancelButton("_Cancel", true), _openButton("_Open", true) { _timeBox.append(_downsampleTimeButton); _downsampleTimeButton.set_active(true); _timeBox.append(_timeDownsampleEntry); _timeDownsampleEntry.set_text("1000"); _box.append(_timeBox); _freqBox.append(_downsampleFreqButton); _downsampleFreqButton.set_active(true); _freqBox.append(_freqDownsampleEntry); _freqDownsampleEntry.set_text("1000"); _box.append(_freqBox); _box.append(_correctHistograms); _buttonBox.append(_cancelButton); _buttonBox.append(_openButton); _openButton.signal_clicked().connect( sigc::mem_fun(*this, &OpenOptionsWindow::onOpen)); _box.append(_buttonBox); set_child(_box); } ~OpenOptionsWindow() {} void ShowForFile(const std::vector& files) { _files = files; present(); } void ShowForFile(const std::string& filename) { _files.clear(); _files.push_back(filename); present(); } sigc::signal&, bool, bool, size_t, size_t, bool)>& SignalOpen() { return _signalOpen; } private: void onOpen() { hide(); size_t timeRes = atol(_timeDownsampleEntry.get_text().c_str()); size_t freqRes = atol(_freqDownsampleEntry.get_text().c_str()); _signalOpen.emit(_files, _downsampleTimeButton.get_active(), _downsampleFreqButton.get_active(), timeRes, freqRes, _correctHistograms.get_active()); _files.clear(); } Gtk::Box _box{Gtk::Orientation::VERTICAL}; Gtk::Box _timeBox; Gtk::CheckButton _downsampleTimeButton; Gtk::Entry _timeDownsampleEntry; Gtk::Box _freqBox; Gtk::CheckButton _downsampleFreqButton; Gtk::Entry _freqDownsampleEntry; Gtk::CheckButton _correctHistograms; Gtk::Box _buttonBox; Gtk::Button _cancelButton, _openButton; sigc::signal&, bool, bool, size_t, size_t, bool)> _signalOpen; std::vector _files; }; #endif aoflagger-v3.5.1/aoqplot/grayscaleplotpage.cpp0000664000175000017500000002345515106657110017663 0ustar oleole#include #include #include "controllers/heatmappagecontroller.h" #include "grayscaleplotpage.h" #include "../rfigui/imagepropertieswindow.h" GrayScalePlotPage::GrayScalePlotPage(HeatMapPageController* controller) : _controller(controller), _selectStatisticKind(QualityTablesFormatter::StandardDeviationStatistic), _imageWidget(), _imagePropertiesWindow(nullptr) { _meanButton.set_group(_countButton); _stdDevButton.set_group(_countButton); _dMeanButton.set_group(_countButton); _dStdDevButton.set_group(_countButton); _rfiPercentageButton.set_group(_countButton); _polPQButton.set_group(_polPPButton); _polQPButton.set_group(_polPPButton); _polQQButton.set_group(_polPPButton); _polIButton.set_group(_polPPButton); _phasePhaseButton.set_group(_amplitudePhaseButton); _realPhaseButton.set_group(_amplitudePhaseButton); _imaginaryPhaseButton.set_group(_amplitudePhaseButton); _rangeWinsorizedButton.set_group(_rangeMinMaxButton); _rangeSpecified.set_group(_rangeMinMaxButton); _winsorNormButton.set_group(_meanNormButton); _winsorNormButton.set_group(_medianNormButton); _imageWidget.SetPlot(controller->Plot()); _imageWidget.set_size_request(300, 300); _imageWidget.set_expand(true); append(_imageWidget); controller->Attach(this); } GrayScalePlotPage::~GrayScalePlotPage() { delete _imagePropertiesWindow; } void GrayScalePlotPage::InitializeToolbar(Gtk::Box& toolbar) { initStatisticKinds(toolbar); initPolarizations(toolbar); initPhaseButtons(toolbar); initPlotOptions(toolbar); } void GrayScalePlotPage::initStatisticKinds(Gtk::Box& toolbar) { toolbar.append(_separator1); _countButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onSelectCount)); _countButton.set_tooltip_text("Visibility count"); toolbar.append(_countButton); _meanButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onSelectMean)); _meanButton.set_tooltip_text("Mean value"); toolbar.append(_meanButton); _stdDevButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onSelectStdDev)); _stdDevButton.set_tooltip_text("Standard deviation"); toolbar.append(_stdDevButton); _dMeanButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onSelectDMean)); _dMeanButton.set_tooltip_text( "Frequency-differential (difference between channels) mean value"); toolbar.append(_dMeanButton); _dStdDevButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onSelectDStdDev)); _dStdDevButton.set_tooltip_text( "Frequency-differential (difference between channels) standard " "deviation"); toolbar.append(_dStdDevButton); _rfiPercentageButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onSelectRFIPercentage)); _rfiPercentageButton.set_tooltip_text("Flagged percentage"); toolbar.append(_rfiPercentageButton); _stdDevButton.set_active(); } void GrayScalePlotPage::initPolarizations(Gtk::Box& toolbar) { toolbar.append(_separator2); _polPPButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::updateImage)); _polPPButton.set_icon_name("showpp"); _polPPButton.set_tooltip_text( "Display statistics for the PP polarization. Depending on the " "polarization configuration of the measurement set, this will show XX or " "RR."); toolbar.append(_polPPButton); _polPQButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::updateImage)); _polPQButton.set_icon_name("showpq"); _polPQButton.set_tooltip_text( "Display statistics for the PQ polarization. Depending on the " "polarization configuration of the measurement set, this will show XY or " "RL."); toolbar.append(_polPQButton); _polQPButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::updateImage)); _polQPButton.set_icon_name("showqp"); _polQPButton.set_tooltip_text( "Display statistics for the QP polarization. Depending on the " "polarization configuration of the measurement set, this will show YX or " "LR."); toolbar.append(_polQPButton); _polQQButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::updateImage)); _polQQButton.set_icon_name("showqq"); _polQQButton.set_tooltip_text( "Display statistics for the QQ polarization. Depending on the " "polarization configuration of the measurement set, this will show YY or " "LL."); toolbar.append(_polQQButton); _polIButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::updateImage)); _polIButton.set_tooltip_text("Stokes I polarization"); toolbar.append(_polIButton); _polIButton.set_active(); } void GrayScalePlotPage::initPhaseButtons(Gtk::Box& toolbar) { toolbar.append(_separator3); _amplitudePhaseButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::updateImage)); _amplitudePhaseButton.set_tooltip_text("Amplitude"); toolbar.append(_amplitudePhaseButton); _phasePhaseButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::updateImage)); _phasePhaseButton.set_tooltip_text("Phase"); toolbar.append(_phasePhaseButton); _realPhaseButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::updateImage)); _realPhaseButton.set_tooltip_text("Real value"); toolbar.append(_realPhaseButton); _imaginaryPhaseButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::updateImage)); _imaginaryPhaseButton.set_tooltip_text("Imaginary value"); toolbar.append(_imaginaryPhaseButton); _amplitudePhaseButton.set_active(); } void GrayScalePlotPage::initPlotOptions(Gtk::Box& toolbar) { toolbar.append(_separator4); _rangeMinMaxButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onSelectMinMaxRange)); toolbar.append(_rangeMinMaxButton); _rangeWinsorizedButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onSelectWinsorizedRange)); toolbar.append(_rangeWinsorizedButton); _rangeSpecified.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onSelectSpecifiedRange)); toolbar.append(_rangeSpecified); toolbar.append(_separator5); _logarithmicScaleButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onLogarithmicScaleClicked)); toolbar.append(_logarithmicScaleButton); _logarithmicScaleButton.set_active(true); _normalizeXAxisButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onNormalizeAxesButtonClicked)); toolbar.append(_normalizeXAxisButton); _normalizeYAxisButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onNormalizeAxesButtonClicked)); toolbar.append(_normalizeYAxisButton); toolbar.append(_separator6); _meanNormButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onChangeNormMethod)); toolbar.append(_meanNormButton); _winsorNormButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onChangeNormMethod)); toolbar.append(_winsorNormButton); _medianNormButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onChangeNormMethod)); toolbar.append(_medianNormButton); _plotPropertiesButton.signal_clicked().connect( sigc::mem_fun(*this, &GrayScalePlotPage::onPropertiesClicked)); toolbar.append(_plotPropertiesButton); } void GrayScalePlotPage::updateImage() { _controller->SetKind(getSelectedStatisticKind()); _controller->SetPolarization(getSelectedPolarization()); _controller->SetPhase(getSelectedPhase()); if (_meanNormButton.get_active()) _controller->SetNormalization(HeatMapPageController::Mean); else if (_winsorNormButton.get_active()) _controller->SetNormalization(HeatMapPageController::Winsorized); else // _medianNormButton _controller->SetNormalization(HeatMapPageController::Median); _controller->UpdateImage(); } aocommon::PolarizationEnum GrayScalePlotPage::getSelectedPolarization() const { if (_polPPButton.get_active()) return aocommon::Polarization::XX; else if (_polPQButton.get_active()) return aocommon::Polarization::XY; else if (_polQPButton.get_active()) return aocommon::Polarization::YX; else if (_polQQButton.get_active()) return aocommon::Polarization::YY; else return aocommon::Polarization::StokesI; } enum TimeFrequencyData::ComplexRepresentation GrayScalePlotPage::getSelectedPhase() const { if (_amplitudePhaseButton.get_active()) return TimeFrequencyData::AmplitudePart; else if (_phasePhaseButton.get_active()) return TimeFrequencyData::PhasePart; else if (_realPhaseButton.get_active()) return TimeFrequencyData::RealPart; else if (_imaginaryPhaseButton.get_active()) return TimeFrequencyData::ImaginaryPart; else return TimeFrequencyData::AmplitudePart; } void GrayScalePlotPage::onPropertiesClicked() { if (_imagePropertiesWindow == nullptr) _imagePropertiesWindow = new ImagePropertiesWindow(_imageWidget, "Plotting properties"); _imagePropertiesWindow->present(); } void GrayScalePlotPage::onSelectMinMaxRange() { _controller->Plot().SetZRange(FullRange()); _imageWidget.Update(); } void GrayScalePlotPage::onSelectWinsorizedRange() { _controller->Plot().SetZRange(WinsorizedRange()); _imageWidget.Update(); } void GrayScalePlotPage::onSelectSpecifiedRange() { RangeConfiguration range = _controller->Plot().ZRange(); range.minimum = RangeLimit::Extreme; range.maximum = RangeLimit::Extreme; _controller->Plot().SetZRange(range); _imageWidget.Update(); } void GrayScalePlotPage::onLogarithmicScaleClicked() { _controller->Plot().SetLogZScale(_logarithmicScaleButton.get_active()); _imageWidget.Update(); } void GrayScalePlotPage::Redraw() { _imageWidget.Update(); } aoflagger-v3.5.1/aoqplot/blengthplotpage.h0000664000175000017500000000212015105361057016764 0ustar oleole#ifndef GUI_QUALITY__BLENGTHPLOTPAGE_H #define GUI_QUALITY__BLENGTHPLOTPAGE_H #include #include "twodimensionalplotpage.h" #include "../structures/msmetadata.h" #include "controllers/blengthpagecontroller.h" class BLengthPlotPage final : public TwoDimensionalPlotPage { public: explicit BLengthPlotPage(BLengthPageController* controller) : TwoDimensionalPlotPage(controller), _controller(controller), _includeAutoCorrelationsButton("Auto-correlations") {} protected: void addCustomPlotButtons(Gtk::Box& container) override { _includeAutoCorrelationsButton.signal_clicked().connect( sigc::mem_fun(*this, &BLengthPlotPage::onAutoCorrelationsClicked)); container.append(_includeAutoCorrelationsButton); _includeAutoCorrelationsButton.show(); } private: void onAutoCorrelationsClicked() { _controller->SetIncludeAutoCorrelations( _includeAutoCorrelationsButton.get_active()); _controller->UpdatePlot(); } BLengthPageController* _controller; Gtk::ToggleButton _includeAutoCorrelationsButton; }; #endif aoflagger-v3.5.1/aoqplot/baselineplotpage.h0000664000175000017500000000067515105043202017125 0ustar oleole#ifndef GUI_QUALITY__BASELINEPLOTPAGE_H #define GUI_QUALITY__BASELINEPLOTPAGE_H #include "../quality/qualitytablesformatter.h" #include "grayscaleplotpage.h" class BaselinePlotPage : public GrayScalePlotPage { public: explicit BaselinePlotPage(class BaselinePageController* controller); virtual ~BaselinePlotPage(); protected: private: class BaselinePageController* _controller; void onMouseMoved(double x, double y); }; #endif aoflagger-v3.5.1/aoqplot/histogrampage.cpp0000664000175000017500000002304015105361057016776 0ustar oleole#include "histogrampage.h" #include #include "controllers/histogrampagecontroller.h" #include "../plot/plotpropertieswindow.h" #include "datawindow.h" HistogramPage::HistogramPage(HistogramPageController* controller) : _controller(controller), _expander("Side bar"), _histogramTypeFrame("Histogram"), _totalHistogramButton("Total"), _rfiHistogramButton("RFI"), _notRFIHistogramButton("Not RFI"), _xxPolarizationButton("XX"), _xyPolarizationButton("XY"), _yxPolarizationButton("YX"), _yyPolarizationButton("YY"), _sumPolarizationButton("Sum"), _fitFrame("Fitting"), _fitButton("Fit"), _subtractFitButton("Subtract"), _fitLogarithmicButton("Log fit"), _fitAutoRangeButton("Auto range"), _functionFrame("Function"), _nsButton("N(S)"), _dndsButton("dN(S)/dS"), _deltaSEntry(), _staircaseFunctionButton("Staircase"), _normalizeButton("Normalize"), _plotPropertiesButton("Properties"), _dataExportButton("Data"), _slopeFrame("Slope"), _drawSlopeButton("Draw"), _drawSlope2Button("Draw2"), _slopeAutoRangeButton("Auto range"), _plotPropertiesWindow(nullptr) { _histogramTypeBox.append(_totalHistogramButton); _totalHistogramButton.set_active(true); _totalHistogramButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _histogramTypeBox.append(_rfiHistogramButton); _rfiHistogramButton.set_active(false); _rfiHistogramButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _histogramTypeBox.append(_notRFIHistogramButton); _notRFIHistogramButton.set_active(false); _notRFIHistogramButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _histogramTypeFrame.set_child(_histogramTypeBox); _sideBox.append(_histogramTypeFrame); _polarizationBox.append(_xxPolarizationButton); _xxPolarizationButton.set_active(false); _xxPolarizationButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _polarizationBox.append(_xyPolarizationButton); _xyPolarizationButton.set_active(false); _xyPolarizationButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _polarizationBox.append(_yxPolarizationButton); _yxPolarizationButton.set_active(false); _yxPolarizationButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _polarizationBox.append(_yyPolarizationButton); _yyPolarizationButton.set_active(false); _yyPolarizationButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _polarizationBox.append(_sumPolarizationButton); _sumPolarizationButton.set_active(true); _sumPolarizationButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _polarizationFrame.set_child(_polarizationBox); _sideBox.append(_polarizationFrame); _fitBox.append(_fitButton); _fitButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _fitBox.append(_subtractFitButton); _subtractFitButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _fitBox.append(_fitLogarithmicButton); _fitLogarithmicButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _fitBox.append(_fitAutoRangeButton); _fitAutoRangeButton.set_active(true); _fitAutoRangeButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::onAutoRangeClicked)); _fitBox.append(_fitStartEntry); _fitStartEntry.set_sensitive(false); _fitStartEntry.signal_activate().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _fitBox.append(_fitEndEntry); _fitEndEntry.set_sensitive(false); _fitEndEntry.signal_activate().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _fitBox.append(_fitTextView); _fitFrame.set_child(_fitBox); _sideBox.append(_fitFrame); _functionBox.append(_nsButton); _nsButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _functionBox.append(_dndsButton); _dndsButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _dndsButton.set_group(_nsButton); _nsButton.set_active(true); _functionBox.append(_deltaSEntry); _deltaSEntry.set_text("2"); _deltaSEntry.signal_activate().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _functionBox.append(_staircaseFunctionButton); _staircaseFunctionButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _functionBox.append(_normalizeButton); _normalizeButton.set_active(true); _normalizeButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _functionFrame.set_child(_functionBox); _sideBox.append(_functionFrame); _plotPropertiesButton.signal_clicked().connect( sigc::mem_fun(*this, &HistogramPage::onPlotPropertiesClicked)); _sideBox.append(_plotPropertiesButton); _dataExportButton.signal_clicked().connect( sigc::mem_fun(*this, &HistogramPage::onDataExportClicked)); _sideBox.append(_dataExportButton); _slopeBox.append(_slopeTextView); _drawSlopeButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _slopeBox.append(_drawSlopeButton); _drawSlope2Button.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _slopeBox.append(_drawSlope2Button); _slopeBox.append(_slopeAutoRangeButton); _slopeAutoRangeButton.set_active(true); _slopeAutoRangeButton.signal_toggled().connect( sigc::mem_fun(*this, &HistogramPage::onSlopeAutoRangeClicked)); _slopeBox.append(_slopeStartEntry); _slopeStartEntry.set_sensitive(false); _slopeStartEntry.signal_activate().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _slopeBox.append(_slopeEndEntry); _slopeEndEntry.set_sensitive(false); _slopeEndEntry.signal_activate().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _slopeBox.append(_slopeRFIRatio); _slopeRFIRatio.set_text("1.0"); _slopeRFIRatio.signal_activate().connect( sigc::mem_fun(*this, &HistogramPage::updatePlot)); _slopeFrame.set_child(_slopeBox); _sideBox.append(_slopeFrame); _expander.set_child(_sideBox); append(_expander); _plotWidget.SetPlot(_controller->Plot()); append(_plotWidget); _plotWidget.set_expand(true); _dataWindow = new DataWindow(); _controller->Attach(this); } HistogramPage::~HistogramPage() { delete _plotPropertiesWindow; delete _dataWindow; } void HistogramPage::updatePlot() { _controller->SetDrawXX(_xxPolarizationButton.get_active()); _controller->SetDrawXY(_xyPolarizationButton.get_active()); _controller->SetDrawYX(_yxPolarizationButton.get_active()); _controller->SetDrawYY(_yyPolarizationButton.get_active()); _controller->SetDrawSum(_sumPolarizationButton.get_active()); _controller->SetAutomaticFitRange(_fitAutoRangeButton.get_active()); _controller->SetFitStart(atof(_fitStartEntry.get_text().c_str())); _controller->SetFitEnd(atof(_fitEndEntry.get_text().c_str())); _controller->SetFitLogarithmic(_fitLogarithmicButton.get_active()); _controller->SetAutomaticSlopeRange(_slopeAutoRangeButton.get_active()); _controller->SetSlopeStart(atof(_slopeStartEntry.get_text().c_str())); _controller->SetSlopeEnd(atof(_slopeEndEntry.get_text().c_str())); _controller->SetSlopeRFIRatio(atof(_slopeRFIRatio.get_text().c_str())); double deltaS = atof(_deltaSEntry.get_text().c_str()); if (deltaS <= 1.0001) deltaS = 1.0001; _controller->SetDeltaS(deltaS); _controller->SetDrawTotal(_totalHistogramButton.get_active()); _controller->SetDrawFit(_fitButton.get_active()); _controller->SetDrawSubtractedFit(_subtractFitButton.get_active()); _controller->SetDrawSlope(_drawSlopeButton.get_active()); _controller->SetDrawSlope2(_drawSlope2Button.get_active()); _controller->SetDrawRFI(_rfiHistogramButton.get_active()); _controller->SetDrawNonRFI(_notRFIHistogramButton.get_active()); _controller->SetDerivative(_dndsButton.get_active()); _controller->SetStaircase(_staircaseFunctionButton.get_active()); _controller->SetNormalize(_normalizeButton.get_active()); _controller->SetDeltaS(atof(_deltaSEntry.get_text().c_str())); } void HistogramPage::onPlotPropertiesClicked() { if (_plotPropertiesWindow == nullptr) { _plotPropertiesWindow = new PlotPropertiesWindow(_controller->Plot(), "Plot properties"); _plotPropertiesWindow->OnChangesApplied = std::bind(&HistogramPage::Redraw, this); } _plotPropertiesWindow->present(); } void HistogramPage::onDataExportClicked() { _dataWindow->present(); updateDataWindow(); } void HistogramPage::SetSlopeFrame(const std::string& str) { _slopeTextView.get_buffer()->set_text(str); if (_slopeAutoRangeButton.get_active()) { double minRange = _controller->SlopeStart(), maxRange = _controller->SlopeEnd(); std::stringstream minRangeStr, maxRangeStr; minRangeStr << minRange; maxRangeStr << maxRange; _slopeStartEntry.set_text(minRangeStr.str()); _slopeEndEntry.set_text(maxRangeStr.str()); } if (_fitAutoRangeButton.get_active()) { std::stringstream minRangeStr, maxRangeStr; minRangeStr << _controller->FitStart(); maxRangeStr << _controller->FitEnd(); _fitStartEntry.set_text(minRangeStr.str()); _fitEndEntry.set_text(maxRangeStr.str()); } } void HistogramPage::updateDataWindow() { if (_dataWindow->get_visible()) _dataWindow->SetData(_controller->Plot()); } aoflagger-v3.5.1/aoqplot/datawindow.cpp0000664000175000017500000000312115105361057016303 0ustar oleole#include "datawindow.h" #include "../plot/xyplot.h" #include #include void DataWindow::SetData(const XYPlot& plot) { _plot = &plot; int selectedIndex = _comboBox.get_active_row_number(); if (selectedIndex < 0) selectedIndex = 0; _comboListStore->clear(); for (size_t i = 0; i < plot.PointSetCount(); ++i) { std::stringstream str; str << (i + 1) << ". " << plot.GetPointSet(i).Label(); Gtk::TreeModel::Row row = *_comboListStore->append(); row[_comboColumnRecord._comboListNameColumn] = str.str(); } if (selectedIndex < (int)plot.PointSetCount()) _comboBox.set_active(selectedIndex); else if (plot.PointSetCount() > 0) _comboBox.set_active(0); onComboChange(); } void DataWindow::onComboChange() { const int active = _comboBox.get_active_row_number(); if (active >= 0) loadData(active); else loadData(_plot->PointSetCount()); } void DataWindow::loadData(size_t plotSetIndex) { std::stringstream _dataStream; _dataStream << std::setprecision(14); if (_plot->PointSetCount() > plotSetIndex) { const XYPointSet& pointSet = _plot->GetPointSet(plotSetIndex); const size_t valueCount = pointSet.Size(); for (size_t i = 0; i < valueCount; ++i) { const double x = pointSet.GetX(i), y = pointSet.GetY(i); if (_plot->XAxis().Type() == AxisType::kText) { const std::string& label = _plot->XAxis().TickLabels()[i].second; _dataStream << i << '\t' << label << '\t' << y << '\n'; } else { _dataStream << i << '\t' << x << '\t' << y << '\n'; } } } SetData(_dataStream.str()); } aoflagger-v3.5.1/aoqplot/twodimensionalplotpage.h0000664000175000017500000000416615106657110020410 0ustar oleole#ifndef GUI_QUALITY__2DPLOTPAGE_H #define GUI_QUALITY__2DPLOTPAGE_H #include #include "controllers/aoqplotpagecontroller.h" #include "../quality/qualitytablesformatter.h" #include "../plot/plotwidget.h" #include "../plot/xyplot.h" #include "plotsheet.h" #include #include #include #include #include #include class TwoDimensionalPlotPage : public PlotSheet { public: explicit TwoDimensionalPlotPage(AOQPlotPageController* _controller); virtual ~TwoDimensionalPlotPage(); void InitializeToolbar(Gtk::Box& toolbar) override final; std::vector GetSelectedKinds() const; std::set GetSelectedPolarizations() const; std::set GetSelectedPhases() const; void Redraw(); protected: virtual void addCustomPlotButtons(Gtk::Box& container) {} private: void updatePlotConfig(); void updateDataWindow(); void initStatisticKindButtons(Gtk::Box& toolbar); void initPolarizationButtons(Gtk::Box& toolbar); void initPhaseButtons(Gtk::Box& toolbar); void initPlotButtons(Gtk::Box& toolbar); void onLogarithmicClicked() { _zeroAxisButton.set_sensitive(!_logarithmicButton.get_active()); updatePlotConfig(); } void onPlotPropertiesClicked(); void onDataExportClicked(); AOQPlotPageController* _controller; Gtk::Separator _separator1, _separator2, _separator3, _separator4; Gtk::ToggleButton _countButton, _meanButton, _stdDevButton, _varianceButton, _dCountButton, _dMeanButton, _dStdDevButton, _rfiPercentageButton; Gtk::ToggleButton _polPPButton, _polPQButton, _polQPButton, _polQQButton, _polIButton; Gtk::ToggleButton _amplitudeButton, _phaseButton, _realButton, _imaginaryButton; Gtk::ToggleButton _logarithmicButton, _zeroAxisButton; Gtk::Button _plotPropertiesButton, _dataExportButton; PlotWidget _plotWidget; std::unique_ptr _plotPropertiesWindow; std::unique_ptr _dataWindow; bool _customButtonsCreated; }; #endif aoflagger-v3.5.1/aoqplot/frequencyplotpage.h0000664000175000017500000000171215105361057017350 0ustar oleole#ifndef GUI_QUALITY__FREQUENCYPLOTPAGE_H #define GUI_QUALITY__FREQUENCYPLOTPAGE_H #include "twodimensionalplotpage.h" #include #include "controllers/frequencypagecontroller.h" #include "../quality/statisticscollection.h" #include "../quality/statisticsderivator.h" class FrequencyPlotPage : public TwoDimensionalPlotPage { public: explicit FrequencyPlotPage(FrequencyPageController* controller) : TwoDimensionalPlotPage(controller), _controller(controller), _ftButton("FT") {} void addCustomPlotButtons(Gtk::Box& container) final { _ftButton.signal_clicked().connect( sigc::mem_fun(*this, &FrequencyPlotPage::onFTButtonClicked)); container.append(_ftButton); _ftButton.show(); } private: void onFTButtonClicked() { _controller->SetPerformFT(_ftButton.get_active()); _controller->UpdatePlot(); } FrequencyPageController* _controller; Gtk::ToggleButton _ftButton; }; #endif aoflagger-v3.5.1/aoqplot/grayscaleplotpage.h0000664000175000017500000000752415106657110017327 0ustar oleole#ifndef GUI_QUALITY__GRAYSCALEPLOTPAGE_H #define GUI_QUALITY__GRAYSCALEPLOTPAGE_H #include #include #include #include #include "../plot/plotwidget.h" #include "../quality/qualitytablesformatter.h" #include "../structures/timefrequencydata.h" #include "../structures/timefrequencymetadata.h" #include "plotsheet.h" class GrayScalePlotPage : public PlotSheet { public: explicit GrayScalePlotPage(class HeatMapPageController* controller); virtual ~GrayScalePlotPage(); void InitializeToolbar(Gtk::Box& toolbar) final; bool NormalizeXAxis() const { return _normalizeXAxisButton.get_active(); } bool NormalizeYAxis() const { return _normalizeYAxisButton.get_active(); } void Redraw(); protected: QualityTablesFormatter::StatisticKind getSelectedStatisticKind() const { return _selectStatisticKind; } void updateImage(); PlotWidget& grayScaleWidget() { return _imageWidget; } private: void initStatisticKinds(Gtk::Box& toolbar); void initPolarizations(Gtk::Box& toolbar); void initPhaseButtons(Gtk::Box& toolbar); void initPlotOptions(Gtk::Box& toolbar); aocommon::PolarizationEnum getSelectedPolarization() const; enum TimeFrequencyData::ComplexRepresentation getSelectedPhase() const; void onSelectCount() { _selectStatisticKind = QualityTablesFormatter::CountStatistic; updateImage(); } void onSelectMean() { _selectStatisticKind = QualityTablesFormatter::MeanStatistic; updateImage(); } void onSelectStdDev() { _selectStatisticKind = QualityTablesFormatter::StandardDeviationStatistic; updateImage(); } void onSelectDCount() { _selectStatisticKind = QualityTablesFormatter::DCountStatistic; updateImage(); } void onSelectDMean() { _selectStatisticKind = QualityTablesFormatter::DMeanStatistic; updateImage(); } void onSelectDStdDev() { _selectStatisticKind = QualityTablesFormatter::DStandardDeviationStatistic; updateImage(); } void onSelectRFIPercentage() { _selectStatisticKind = QualityTablesFormatter::RFIPercentageStatistic; updateImage(); } void onSelectSNR() { _selectStatisticKind = QualityTablesFormatter::SignalToNoiseStatistic; updateImage(); } void onPropertiesClicked(); void onSelectMinMaxRange(); void onSelectWinsorizedRange(); void onSelectSpecifiedRange(); void onLogarithmicScaleClicked(); void onNormalizeAxesButtonClicked() { updateImage(); } void onChangeNormMethod() { if (_normalizeYAxisButton.get_active()) updateImage(); } class HeatMapPageController* _controller; Gtk::Separator _separator1, _separator2, _separator3, _separator4, _separator5, _separator6; Gtk::ToggleButton _countButton{"#"}; Gtk::ToggleButton _meanButton{"μ"}; Gtk::ToggleButton _stdDevButton{"σ"}; Gtk::ToggleButton _dMeanButton{"Δμ"}; Gtk::ToggleButton _dStdDevButton{"Δσ"}; Gtk::ToggleButton _rfiPercentageButton{"%"}; Gtk::ToggleButton _polPPButton{"pp"}, _polPQButton{"pq"}, _polQPButton{"qp"}, _polQQButton{"qq"}, _polIButton{"I"}; Gtk::ToggleButton _amplitudePhaseButton{"A"}; Gtk::ToggleButton _phasePhaseButton{"ϕ"}; Gtk::ToggleButton _realPhaseButton{"r"}; Gtk::ToggleButton _imaginaryPhaseButton{"i"}; Gtk::ToggleButton _rangeMinMaxButton{"M"}; Gtk::ToggleButton _rangeWinsorizedButton{"W"}; Gtk::ToggleButton _rangeSpecified{"S"}; Gtk::ToggleButton _logarithmicScaleButton{"L"}; Gtk::ToggleButton _normalizeXAxisButton{"NX"}; Gtk::ToggleButton _normalizeYAxisButton{"NY"}; Gtk::ToggleButton _meanNormButton{"Nμ"}; Gtk::ToggleButton _winsorNormButton{"NW"}; Gtk::ToggleButton _medianNormButton{"NM"}; Gtk::Button _plotPropertiesButton{"P"}; QualityTablesFormatter::StatisticKind _selectStatisticKind; PlotWidget _imageWidget; class ImagePropertiesWindow* _imagePropertiesWindow; }; #endif aoflagger-v3.5.1/aoqplot/twodimensionalplotpage.cpp0000664000175000017500000002400615106657110020736 0ustar oleole#include #include #include "datawindow.h" #include "twodimensionalplotpage.h" #include "../quality/statisticscollection.h" #include "../quality/statisticsderivator.h" #include "../plot/plotpropertieswindow.h" #include TwoDimensionalPlotPage::TwoDimensionalPlotPage( AOQPlotPageController* controller) : _controller(controller), _countButton("#"), _meanButton("μ"), _stdDevButton("σ"), _varianceButton("σ²"), _dCountButton("Δ#"), _dMeanButton("Δμ"), _dStdDevButton("Δσ"), _rfiPercentageButton("%"), _polPPButton("pp"), _polPQButton("pq"), _polQPButton("qp"), _polQQButton("qq"), _polIButton("I"), _amplitudeButton("A"), _phaseButton("ϕ"), _realButton("r"), _imaginaryButton("i"), _logarithmicButton("Log"), _zeroAxisButton("0"), _plotPropertiesButton("P"), _dataExportButton("D"), _plotPropertiesWindow(nullptr), _dataWindow(new DataWindow()), _customButtonsCreated(false) { _plotWidget.SetPlot(_controller->Plot()); _plotWidget.set_expand(true); append(_plotWidget); _controller->Attach(this); } TwoDimensionalPlotPage::~TwoDimensionalPlotPage() = default; void TwoDimensionalPlotPage::updatePlotConfig() { _controller->Plot().SetIncludeZeroYAxis(_zeroAxisButton.get_active()); _controller->Plot().YAxis().SetLogarithmic(_logarithmicButton.get_active()); _plotWidget.Update(); } std::vector TwoDimensionalPlotPage::GetSelectedKinds() const { std::vector kinds; if (_countButton.get_active()) kinds.emplace_back(QualityTablesFormatter::CountStatistic); if (_meanButton.get_active()) kinds.emplace_back(QualityTablesFormatter::MeanStatistic); if (_stdDevButton.get_active()) kinds.emplace_back(QualityTablesFormatter::StandardDeviationStatistic); if (_varianceButton.get_active()) kinds.emplace_back(QualityTablesFormatter::VarianceStatistic); if (_dCountButton.get_active()) kinds.emplace_back(QualityTablesFormatter::DCountStatistic); if (_dMeanButton.get_active()) kinds.emplace_back(QualityTablesFormatter::DMeanStatistic); if (_dStdDevButton.get_active()) kinds.emplace_back(QualityTablesFormatter::DStandardDeviationStatistic); if (_rfiPercentageButton.get_active()) kinds.emplace_back(QualityTablesFormatter::RFIPercentageStatistic); return kinds; } std::set TwoDimensionalPlotPage::GetSelectedPolarizations() const { std::set pols; if (_polPPButton.get_active()) pols.insert(AOQPlotPageController::PolPP); if (_polPQButton.get_active()) pols.insert(AOQPlotPageController::PolPQ); if (_polQPButton.get_active()) pols.insert(AOQPlotPageController::PolQP); if (_polQQButton.get_active()) pols.insert(AOQPlotPageController::PolQQ); if (_polIButton.get_active()) pols.insert(AOQPlotPageController::PolI); return pols; } std::set TwoDimensionalPlotPage::GetSelectedPhases() const { std::set phases; if (_amplitudeButton.get_active()) phases.insert(AOQPlotPageController::AmplitudePhaseType); if (_phaseButton.get_active()) phases.insert(AOQPlotPageController::PhasePhaseType); if (_realButton.get_active()) phases.insert(AOQPlotPageController::RealPhaseType); if (_imaginaryButton.get_active()) phases.insert(AOQPlotPageController::ImaginaryPhaseType); return phases; } void TwoDimensionalPlotPage::InitializeToolbar(Gtk::Box& toolbar) { initStatisticKindButtons(toolbar); initPolarizationButtons(toolbar); initPhaseButtons(toolbar); initPlotButtons(toolbar); if (!_customButtonsCreated) { addCustomPlotButtons(toolbar); _customButtonsCreated = true; } _controller->UpdatePlot(); } void TwoDimensionalPlotPage::initStatisticKindButtons(Gtk::Box& toolbar) { toolbar.append(_separator1); _countButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _countButton.set_tooltip_text("Visibility count"); toolbar.append(_countButton); _meanButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _meanButton.set_tooltip_text("Mean value"); toolbar.append(_meanButton); _stdDevButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _stdDevButton.set_active(true); _stdDevButton.set_tooltip_text("Standard deviation"); toolbar.append(_stdDevButton); _varianceButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); toolbar.append(_varianceButton); //_dCountButton.signal_clicked().connect(sigc::mem_fun(*_controller, //&AOQPlotPageController::UpdatePlot)); toolbar.append(_dCountButton); _dMeanButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _dMeanButton.set_tooltip_text( "Frequency-differential (difference between channels) mean value"); toolbar.append(_dMeanButton); _dStdDevButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _dStdDevButton.set_tooltip_text( "Frequency-differential (difference between channels) standard " "deviation"); toolbar.append(_dStdDevButton); _rfiPercentageButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _rfiPercentageButton.set_tooltip_text("Flagged percentage"); toolbar.append(_rfiPercentageButton); } void TwoDimensionalPlotPage::initPolarizationButtons(Gtk::Box& toolbar) { toolbar.append(_separator2); _polPPButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _polPPButton.set_active(false); _polPPButton.set_icon_name("showpp"); _polPPButton.set_tooltip_text( "Display statistics for the PP polarization. Depending on the " "polarization configuration of the measurement set, this will show XX or " "RR."); toolbar.append(_polPPButton); _polPQButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _polPQButton.set_active(false); _polPQButton.set_icon_name("showpq"); _polPQButton.set_tooltip_text( "Display statistics for the PQ polarization. Depending on the " "polarization configuration of the measurement set, this will show XY or " "RL."); toolbar.append(_polPQButton); _polQPButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _polQPButton.set_active(false); _polQPButton.set_icon_name("showqp"); _polPQButton.set_tooltip_text( "Display statistics for the QP polarization. Depending on the " "polarization configuration of the measurement set, this will show YX or " "LR."); toolbar.append(_polQPButton); _polQQButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _polQQButton.set_active(false); _polQQButton.set_icon_name("showqq"); _polQQButton.set_tooltip_text( "Display statistics for the QQ polarization. Depending on the " "polarization configuration of the measurement set, this will show YY or " "LL."); toolbar.append(_polQQButton); _polIButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _polIButton.set_active(true); _polIButton.set_tooltip_text("Display statistics for QQ + PP."); toolbar.append(_polIButton); } void TwoDimensionalPlotPage::initPhaseButtons(Gtk::Box& toolbar) { toolbar.append(_separator3); _amplitudeButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _amplitudeButton.set_active(true); _amplitudeButton.set_tooltip_text("Amplitude"); toolbar.append(_amplitudeButton); _phaseButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _phaseButton.set_tooltip_text("Phase"); toolbar.append(_phaseButton); _realButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _realButton.set_tooltip_text("Real value"); toolbar.append(_realButton); _imaginaryButton.signal_clicked().connect( sigc::mem_fun(*_controller, &AOQPlotPageController::UpdatePlot)); _imaginaryButton.set_tooltip_text("Imaginary value"); toolbar.append(_imaginaryButton); } void TwoDimensionalPlotPage::initPlotButtons(Gtk::Box& toolbar) { toolbar.append(_separator4); _logarithmicButton.signal_clicked().connect( sigc::mem_fun(*this, &TwoDimensionalPlotPage::onLogarithmicClicked)); toolbar.append(_logarithmicButton); _zeroAxisButton.signal_clicked().connect( sigc::mem_fun(*this, &TwoDimensionalPlotPage::updatePlotConfig)); _zeroAxisButton.set_active(true); toolbar.append(_zeroAxisButton); _controller->Plot().SetIncludeZeroYAxis(true); _plotPropertiesButton.signal_clicked().connect( sigc::mem_fun(*this, &TwoDimensionalPlotPage::onPlotPropertiesClicked)); toolbar.append(_plotPropertiesButton); _dataExportButton.signal_clicked().connect( sigc::mem_fun(*this, &TwoDimensionalPlotPage::onDataExportClicked)); toolbar.append(_dataExportButton); } void TwoDimensionalPlotPage::onPlotPropertiesClicked() { if (_plotPropertiesWindow == nullptr) { _plotPropertiesWindow.reset( new PlotPropertiesWindow(_controller->Plot(), "Plot properties")); _plotPropertiesWindow->OnChangesApplied = std::bind(&AOQPlotPageController::UpdatePlot, _controller); } _plotPropertiesWindow->present(); } void TwoDimensionalPlotPage::updateDataWindow() { if (_dataWindow->get_visible()) _dataWindow->SetData(_controller->Plot()); } void TwoDimensionalPlotPage::onDataExportClicked() { _dataWindow->present(); updateDataWindow(); } void TwoDimensionalPlotPage::Redraw() { _plotWidget.Update(); if (_dataWindow->get_visible()) { updateDataWindow(); } } aoflagger-v3.5.1/aoqplot/aoqplotwindow.h0000664000175000017500000000407315106657110016524 0ustar oleole#ifndef AOQPLOT_WINDOW_H #define AOQPLOT_WINDOW_H #include #include #include #include #include #include #include #include #include #include #include #include #include "../quality/qualitytablesformatter.h" #include "plotsheet.h" #include "openoptionswindow.h" #include "../structures/antennainfo.h" #include "controllers/aoqpagecontroller.h" class AOQPlotWindow : public Gtk::ApplicationWindow { public: explicit AOQPlotWindow(class AOQPlotController* controller); void Open(const std::vector& files); void Open(const std::string& file) { std::vector files(1); files[0] = file; Open(files); } void SetStatus(const std::string& newStatus) { onStatusChange(newStatus); } void ShowError(const std::string& message); void SetShowHistograms(bool show) { /*_histogramMI.set_sensitive(show);*/ } private: void onOpenOptionsSelected(const std::vector& files, bool downsampleTime, bool downsampleFreq, size_t timeSize, size_t freqSize, bool correctHistograms); void close(); void readDistributedObservation(const std::string& filename, bool correctHistograms); void readMetaInfoFromMS(const std::string& filename); void readAndCombine(const std::string& filename); void onStatusChange(const std::string& newStatus); void onChangeSheet(int new_sheet); class AOQPlotController* _controller; int _activeSheetIndex; Gtk::Box toolbar_; Gtk::MenuButton page_menu_button_; std::shared_ptr page_selection_; Gtk::Box _vBox{Gtk::Orientation::VERTICAL}; Gtk::Statusbar _statusBar; std::unique_ptr _pageController; std::unique_ptr plot_page_; OpenOptionsWindow _openOptionsWindow; std::shared_ptr dialog_; }; #endif aoflagger-v3.5.1/aoqplot/datawindow.h0000664000175000017500000000305315105361057015754 0ustar oleole#ifndef GUI_QUALITY__DATA_WINDOW_H #define GUI_QUALITY__DATA_WINDOW_H #include #include #include #include #include #include #include class XYPlot; class DataWindow : public Gtk::Window { public: DataWindow() { _box.append(_comboBox); _comboListStore = Gtk::ListStore::create(_comboColumnRecord); _comboBox.set_model(_comboListStore); _comboBox.pack_start(_comboColumnRecord._comboListNameColumn); _comboBox.signal_changed().connect( sigc::mem_fun(*this, &DataWindow::onComboChange)); _comboBox.show(); _scrolledWindow.set_child(_textView); _textView.show(); _box.append(_scrolledWindow); _scrolledWindow.show(); set_child(_box); _box.show(); set_default_size(300, 400); } ~DataWindow() {} void SetData(const std::string& data) { _textView.get_buffer()->set_text(data); } void SetData(const XYPlot& plot); private: DataWindow(const DataWindow& dataWindow) = delete; void onComboChange(); void loadData(size_t plotSetIndex); class ComboColumnRecord : public Gtk::TreeModel::ColumnRecord { public: ComboColumnRecord() { add(_comboListNameColumn); } Gtk::TreeModelColumn _comboListNameColumn; } _comboColumnRecord; Gtk::Box _box{Gtk::Orientation::VERTICAL}; Gtk::ComboBox _comboBox; Glib::RefPtr _comboListStore; Gtk::ScrolledWindow _scrolledWindow; Gtk::TextView _textView; const XYPlot* _plot; }; #endif aoflagger-v3.5.1/aoqplot/controllers/0000775000175000017500000000000015146315735016016 5ustar oleoleaoflagger-v3.5.1/aoqplot/controllers/polbuttonsetcontroller.h0000664000175000017500000000103614752462134023033 0ustar oleole#ifndef POL_BUTTON_SET_CONTROLLER_H #define POL_BUTTON_SET_CONTROLLER_H class PolButtonSetController { public: PolButtonSetController() : _ppSelected(true), _pqSelected(false), _qpSelected(false), _qqSelected(true) {} bool IsPPSelected() const { return _ppSelected; } bool IsPQSelected() const { return _pqSelected; } bool IsQPSelected() const { return _qpSelected; } bool IsQQSelected() const { return _qqSelected; } private: bool _ppSelected, _pqSelected, _qpSelected, _qqSelected; }; #endif aoflagger-v3.5.1/aoqplot/controllers/heatmappagecontroller.cpp0000664000175000017500000000733415105361057023102 0ustar oleole#include "heatmappagecontroller.h" #include "../../quality/statisticscollection.h" #include "../../quality/statisticsderivator.h" #include "../grayscaleplotpage.h" #include "../../structures/samplerow.h" HeatMapPageController::HeatMapPageController() : _page(nullptr), _statisticKind(QualityTablesFormatter::StandardDeviationStatistic), _polarization(aocommon::Polarization::StokesI), _phase(TimeFrequencyData::AmplitudePart) { _heatMap.SetCairoFilter(Cairo::SurfacePattern::Filter::NEAREST); _heatMap.SetColorMap(ColorMap::HotCold); _heatMap.SetZRange(FullRange()); _heatMap.SetLogZScale(true); _heatMap.SetZAxisDescription("Statistical value"); _heatMap.SetManualZAxisDescription(true); } void HeatMapPageController::updateImageImpl( QualityTablesFormatter::StatisticKind statisticKind, aocommon::PolarizationEnum polarisation, enum TimeFrequencyData::ComplexRepresentation phase) { std::pair pair = constructImage(statisticKind); TimeFrequencyData& data = pair.first; if (!data.IsEmpty()) { setToPolarization(data, polarisation); setToPhase(data, phase); Image2DCPtr image = data.GetSingleImage(); if (_page != nullptr) { if (_page->NormalizeXAxis()) image = Image2D::MakePtr(normalizeXAxis(*image)); if (_page->NormalizeYAxis()) image = Image2D::MakePtr(normalizeYAxis(*image)); } _heatMap.SetZAxisDescription( StatisticsDerivator::GetDescWithUnits(statisticKind)); _heatMap.SetImage(std::unique_ptr(new PlotImage(image))); _heatMap.SetOriginalMask(data.GetSingleMask()); if (pair.second != nullptr) _heatMap.SetMetaData(pair.second); if (_page != nullptr) _page->Redraw(); } } Image2D HeatMapPageController::normalizeXAxis(const Image2D& input) { Image2D output = Image2D::MakeUnsetImage(input.Width(), input.Height()); for (size_t x = 0; x < input.Width(); ++x) { const SampleRow row = SampleRow::MakeFromColumn(&input, x); num_t norm; if (_normalization == Mean) norm = 1.0 / row.MeanWithMissings(); else if (_normalization == Winsorized) norm = 1.0 / row.WinsorizedMeanWithMissings(); else // _medianNormButton norm = 1.0 / row.MedianWithMissings(); for (size_t y = 0; y < input.Height(); ++y) output.SetValue(x, y, input.Value(x, y) * norm); } return output; } Image2D HeatMapPageController::normalizeYAxis(const Image2D& input) { Image2D output = Image2D::MakeUnsetImage(input.Width(), input.Height()); for (size_t y = 0; y < input.Height(); ++y) { const SampleRow row = SampleRow::MakeFromRow(&input, y); num_t norm; if (_normalization == Mean) norm = 1.0 / row.MeanWithMissings(); else if (_normalization == Winsorized) norm = 1.0 / row.WinsorizedMeanWithMissings(); else // _medianNormButton norm = 1.0 / row.MedianWithMissings(); for (size_t x = 0; x < input.Width(); ++x) output.SetValue(x, y, input.Value(x, y) * norm); } return output; } void HeatMapPageController::setToPolarization( TimeFrequencyData& data, aocommon::PolarizationEnum polarisation) { if ((polarisation == aocommon::Polarization::StokesI && data.HasPolarization(aocommon::Polarization::XX) && data.HasPolarization(aocommon::Polarization::YY)) || (polarisation != aocommon::Polarization::StokesI && data.HasPolarization(polarisation))) { data = data.Make(polarisation); if (polarisation == aocommon::Polarization::StokesI) data.MultiplyImages(0.5); } else { data.SetImagesToZero(); } } void HeatMapPageController::setToPhase( TimeFrequencyData& data, enum TimeFrequencyData::ComplexRepresentation phase) { data = data.Make(phase); } aoflagger-v3.5.1/aoqplot/controllers/blengthpagecontroller.h0000664000175000017500000000430514752462134022553 0ustar oleole#ifndef BLENGTH_PATH_CONTROLLER_H #define BLENGTH_PATH_CONTROLLER_H #include #include #include #include #include "aoqplotpagecontroller.h" #include "../../quality/baselinestatisticsmap.h" #include "../../quality/statisticscollection.h" class BLengthPageController final : public AOQPlotPageController { public: void SetIncludeAutoCorrelations(bool inclAutos) { _includeAutoCorrelations = inclAutos; } protected: void processStatistics(const StatisticsCollection* statCollection, const std::vector& antennas) override { _statisticsWithAutocorrelations.clear(); _statisticsWithoutAutocorrelations.clear(); const BaselineStatisticsMap& map = statCollection->BaselineStatistics(); std::vector> baselines = map.BaselineList(); for (std::vector>::const_iterator i = baselines.begin(); i != baselines.end(); ++i) { Baseline bline(antennas[i->first], antennas[i->second]); const DefaultStatistics& statistics = map.GetStatistics(i->first, i->second); _statisticsWithAutocorrelations.insert( std::pair(bline.Distance(), statistics)); if (i->first != i->second) _statisticsWithoutAutocorrelations.insert( std::pair(bline.Distance(), statistics)); } } const std::map& getStatistics() const override { return _includeAutoCorrelations ? _statisticsWithAutocorrelations : _statisticsWithoutAutocorrelations; } void startLine(XYPlot& plot, const std::string& name, int lineIndex, const std::string& yAxisDesc, bool second_axis) override { XYPointSet& points = plot.StartLine(name, "Baseline length (m)", yAxisDesc, XYPointSet::DrawPoints); points.SetUseSecondYAxis(second_axis); } private: bool _includeAutoCorrelations = false; std::map _statisticsWithAutocorrelations; std::map _statisticsWithoutAutocorrelations; }; #endif aoflagger-v3.5.1/aoqplot/controllers/baselinepagecontroller.cpp0000664000175000017500000000651414752462134023251 0ustar oleole#include "baselinepagecontroller.h" #include "../../quality/statisticscollection.h" #include "../../quality/statisticsderivator.h" using aocommon::Polarization; std::pair BaselinePageController::constructImage( QualityTablesFormatter::StatisticKind kind) { if (HasStatistics()) { const unsigned polarizationCount = _statCollection->PolarizationCount(); std::vector> baselines = _statCollection->BaselineStatistics().BaselineList(); const StatisticsDerivator derivator(*_statCollection); const unsigned antennaCount = _statCollection->BaselineStatistics().AntennaCount(); std::vector realImages(polarizationCount); std::vector imagImages(polarizationCount); std::vector mask(polarizationCount); for (unsigned p = 0; p < polarizationCount; ++p) { realImages[p] = Image2D::CreateUnsetImagePtr(antennaCount, antennaCount); realImages[p]->SetAll(std::numeric_limits::quiet_NaN()); imagImages[p] = Image2D::CreateUnsetImagePtr(antennaCount, antennaCount); imagImages[p]->SetAll(std::numeric_limits::quiet_NaN()); mask[p] = Mask2D::CreateSetMaskPtr(antennaCount, antennaCount); } for (std::vector>::const_iterator i = baselines.begin(); i != baselines.end(); ++i) { const unsigned antenna1 = i->first, antenna2 = i->second; for (unsigned p = 0; p < polarizationCount; ++p) { const std::complex val = derivator.GetComplexBaselineStatistic(kind, antenna1, antenna2, p); realImages[p]->SetValue(antenna1, antenna2, val.real()); imagImages[p]->SetValue(antenna1, antenna2, val.imag()); mask[p]->SetValue(antenna1, antenna2, false); } } TimeFrequencyData data; if (polarizationCount == 1) { data = TimeFrequencyData(Polarization::StokesI, realImages[0], imagImages[0]); data.SetGlobalMask(mask[0]); } else if (polarizationCount == 2) { data = TimeFrequencyData(Polarization::XX, realImages[0], imagImages[0], Polarization::YY, realImages[1], imagImages[1]); data.SetIndividualPolarizationMasks(mask[0], mask[1]); } else if (polarizationCount == 4) { data = TimeFrequencyData::FromLinear( realImages[0], imagImages[0], realImages[1], imagImages[1], realImages[2], imagImages[2], realImages[3], imagImages[3]); data.SetIndividualPolarizationMasks(mask[0], mask[1], mask[2], mask[3]); } else { std::stringstream s; s << "Set has not 1, 2 or 4 polarizations (?!?) : " "StatisticsCollection.PolarizationCount() == " << polarizationCount; throw std::runtime_error(s.str()); } return std::pair( data, TimeFrequencyMetaDataCPtr()); } else { return std::pair( TimeFrequencyData(), TimeFrequencyMetaDataCPtr()); } } std::string BaselinePageController::AntennaName(size_t index) const { std::string name; if (_antennas == nullptr) { std::stringstream s; s << index; name = s.str(); } else { name = (*_antennas)[index].name; } return name; } aoflagger-v3.5.1/aoqplot/controllers/aoqplotcontroller.cpp0000664000175000017500000001172315106657110022301 0ustar oleole#include "aoqplotcontroller.h" #include "antennapagecontroller.h" #include "baselinepagecontroller.h" #include "blengthpagecontroller.h" #include "frequencypagecontroller.h" #include "tfpagecontroller.h" #include "timepagecontroller.h" #include "../aoqplotwindow.h" #include "../baselineplotpage.h" #include "../blengthplotpage.h" #include "../frequencyplotpage.h" #include "../histogrampage.h" #include "../plotsheet.h" #include "../summarypage.h" #include "../timefrequencyplotpage.h" #include "../../structures/msmetadata.h" #include "../../quality/combine.h" #include "../../quality/histogramtablesformatter.h" #include "../../quality/histogramcollection.h" #include "../../quality/statisticscollection.h" AOQPlotController::AOQPlotController() : _isOpen(false), _window(nullptr) {} AOQPlotController::~AOQPlotController() = default; void AOQPlotController::close() { if (_isOpen) { _statCollection.reset(); _histCollection.reset(); _fullStats.reset(); _isOpen = false; } } void AOQPlotController::readMetaInfoFromMS(const string& filename) { const MSMetaData ms(filename); _polarizationCount = ms.PolarizationCount(); const unsigned antennaCount = ms.AntennaCount(); _antennas.clear(); for (unsigned a = 0; a < antennaCount; ++a) _antennas.push_back(ms.GetAntennaInfo(a)); } void AOQPlotController::ReadStatistics(const std::vector& files, bool downsampleTime, bool downsampleFreq, size_t timeSize, size_t freqSize, bool correctHistograms) { close(); if (!files.empty()) { const std::string& firstFile = *files.begin(); readMetaInfoFromMS(firstFile); quality::FileContents contents = quality::ReadAndCombine(files, true); _statCollection = std::make_unique( std::move(contents.statistics_collection)); _histCollection = std::make_unique( std::move(contents.histogram_collection)); if (_window != nullptr) _window->SetShowHistograms(!_histCollection->Empty()); if (downsampleTime) { std::cout << "Lowering time resolution..." << std::endl; _statCollection->LowerTimeResolution(timeSize); } if (downsampleFreq) { std::cout << "Lowering frequency resolution..." << std::endl; _statCollection->LowerFrequencyResolution(freqSize); } std::cout << "Integrating baseline statistics to one channel..." << std::endl; _statCollection->IntegrateBaselinesToOneChannel(); std::cout << "Regridding time statistics..." << std::endl; _statCollection->RegridTime(); std::cout << "Copying statistics..." << std::endl; _fullStats = std::make_unique(*_statCollection); std::cout << "Integrating time statistics to one channel..." << std::endl; _statCollection->IntegrateTimeToOneChannel(); std::cout << "Opening statistics panel..." << std::endl; _isOpen = true; } } void AOQPlotController::Save(const AOQPlotController::PlotSavingData& data, size_t width, size_t height) { const std::string& prefix = data.filenamePrefix; const QualityTablesFormatter::StatisticKind kind = data.statisticKind; std::cout << "Saving " << prefix << "-antennas.pdf...\n"; AntennaePageController antController; antController.SetStatistics(_statCollection.get(), _antennas); antController.SavePdf(prefix + "-antennas.pdf", kind); std::cout << "Saving " << prefix << "-baselines.pdf...\n"; BaselinePageController baselController; // BaselinePlotPage baselPage(&baselController); baselController.SetStatistics(_statCollection.get(), _antennas); baselController.SavePdf(prefix + "-baselines.pdf", kind, width, height); std::cout << "Saving " << prefix << "-baselinelengths.pdf...\n"; BLengthPageController blenController; blenController.SetStatistics(_statCollection.get(), _antennas); blenController.SavePdf(prefix + "-baselinelengths.pdf", kind); std::cout << "Saving " << prefix << "-timefrequency.pdf...\n"; TFPageController tfController; tfController.SetStatistics(_fullStats.get(), _antennas); tfController.SavePdf(prefix + "-timefrequency.pdf", kind, width, height); std::cout << "Saving " << prefix << "-time.pdf...\n"; TimePageController timeController; timeController.SetStatistics(_statCollection.get(), _antennas); timeController.SavePdf(prefix + "-time.pdf", kind); std::cout << "Saving " << prefix << "-frequency.pdf...\n"; FrequencyPageController freqController; freqController.SetStatistics(_statCollection.get(), _antennas); freqController.SavePdf(prefix + "-frequency.pdf", kind); } void AOQPlotController::Initialize(AOQPageController* controller, bool averagedStats) { if (averagedStats) controller->SetStatistics(_statCollection.get(), _antennas); else controller->SetStatistics(_fullStats.get(), _antennas); controller->SetHistograms(_histCollection.get()); } aoflagger-v3.5.1/aoqplot/controllers/antennapagecontroller.h0000664000175000017500000000422014752462134022550 0ustar oleole#ifndef ANTENNAE_PAGE_CONTROLLER_H #define ANTENNAE_PAGE_CONTROLLER_H #include "aoqplotpagecontroller.h" #include #include #include #include #include "../../quality/statisticscollection.h" #include "../../structures/msmetadata.h" class AntennaePageController final : public AOQPlotPageController { protected: void processStatistics(const StatisticsCollection* statCollection, const std::vector& antennas) override { _antennas = antennas; const BaselineStatisticsMap& map = statCollection->BaselineStatistics(); std::vector> baselines = map.BaselineList(); for (std::vector>::const_iterator i = baselines.begin(); i != baselines.end(); ++i) { if (i->first != i->second) { const DefaultStatistics& stats = map.GetStatistics(i->first, i->second); addStatistic(i->first, stats); addStatistic(i->second, stats); } } } const std::map& getStatistics() const override { return _statistics; } void startLine(XYPlot& plot, const std::string& name, int lineIndex, const std::string& yAxisDesc, bool second_axis) override { XYPointSet& points = plot.StartLine(name, "Antenna index", yAxisDesc, XYPointSet::DrawColumns); points.SetUseSecondYAxis(second_axis); std::vector> labels; for (size_t i = 0; i != _antennas.size(); ++i) labels.emplace_back(static_cast(i), _antennas[i].name); plot.XAxis().SetTickLabels(std::move(labels)); plot.XAxis().SetRotateUnits(true); } void addStatistic(unsigned antIndex, const DefaultStatistics& stats) { std::map::iterator iter = _statistics.find(antIndex); if (iter == _statistics.end()) _statistics.insert(std::pair(antIndex, stats)); else iter->second += stats; } private: std::map _statistics; std::vector _antennas; }; #endif aoflagger-v3.5.1/aoqplot/controllers/baselinepagecontroller.h0000664000175000017500000000251015002644503022675 0ustar oleole#ifndef BASELINE_PAGE_CONTROLLER_H #define BASELINE_PAGE_CONTROLLER_H #include "heatmappagecontroller.h" #include #include #include class AntennaInfo; class BaselinePageController final : public HeatMapPageController { public: BaselinePageController() : _statCollection(nullptr), _antennas(nullptr) {} void SetStatistics(const StatisticsCollection* statCollection, const std::vector& antennas) override { _statCollection = statCollection; _antennas = &antennas; HeatMap& map = Plot(); map.SetXAxisDescription("Antenna 1 index"); map.SetXAxisMin(0); map.SetXAxisMax(antennas.size()); map.SetIntegerXAxis(true); map.SetYAxisDescription("Antenna 2 index"); map.SetYAxisMin(0); map.SetYAxisMax(antennas.size()); map.SetIntegerYAxis(true); UpdateImage(); } void CloseStatistics() override { _statCollection = nullptr; _antennas = nullptr; } bool HasStatistics() const { return _statCollection != nullptr; } std::string AntennaName(size_t index) const; protected: std::pair constructImage( QualityTablesFormatter::StatisticKind kind) override; private: const StatisticsCollection* _statCollection; const std::vector* _antennas; }; #endif aoflagger-v3.5.1/aoqplot/controllers/aoqplotcontroller.h0000664000175000017500000000254014752462134021751 0ustar oleole#ifndef AOQPLOT_CONTROLLER_H #define AOQPLOT_CONTROLLER_H #include "../../structures/antennainfo.h" #include "../../quality/qualitytablesformatter.h" #include #include #include #include class AOQPlotController { public: AOQPlotController(); ~AOQPlotController(); void Attach(class AOQPlotWindow* window) { _window = window; } void ReadStatistics(const std::vector& files) { ReadStatistics(files, true, true, 1000, 1000, false); } void ReadStatistics(const std::vector& files, bool downsampleTime, bool downsampleFreq, size_t timeSize, size_t freqSize, bool correctHistograms); struct PlotSavingData { QualityTablesFormatter::StatisticKind statisticKind; std::string filenamePrefix; }; void Save(const PlotSavingData& data, size_t width, size_t height); void Initialize(class AOQPageController* controller, bool averagedStats); private: void close(); void readMetaInfoFromMS(const std::string& filename); bool _isOpen; std::unique_ptr _statCollection; std::unique_ptr _histCollection; std::unique_ptr _fullStats; std::vector _antennas; size_t _polarizationCount; class AOQPlotWindow* _window; }; #endif aoflagger-v3.5.1/aoqplot/controllers/aoqplotpagecontroller.h0000664000175000017500000000423614752462134022612 0ustar oleole#ifndef AOQ_PLOT_PAGE_CONTROLLER #define AOQ_PLOT_PAGE_CONTROLLER #include #include #include #include #include "../../quality/qualitytablesformatter.h" #include "../../plot/xyplot.h" #include "aoqpagecontroller.h" class AOQPlotPageController : public AOQPageController { public: enum SelectedPol { PolPP, PolPQ, PolQP, PolQQ, PolI }; AOQPlotPageController(); void Attach(class TwoDimensionalPlotPage* page) { _page = page; } void SetStatistics( const StatisticsCollection* statCollection, const std::vector& antennas) override final; void CloseStatistics() override final { _statCollection = nullptr; } bool HasStatistics() const { return _statCollection != nullptr; } void SavePdf(const std::string& filename, QualityTablesFormatter::StatisticKind kind); XYPlot& Plot() { return _plot; } void UpdatePlot(); enum PhaseType { AmplitudePhaseType, PhasePhaseType, RealPhaseType, ImaginaryPhaseType }; protected: virtual void processStatistics(const StatisticsCollection*, const std::vector&) {} virtual const std::map& getStatistics() const = 0; virtual void startLine(XYPlot& plot, const std::string& name, int lineIndex, const std::string& yAxisDesc, bool second_axis) = 0; virtual void processPlot(XYPlot& plot) {} const StatisticsCollection* getStatCollection() const { return _statCollection; } class TwoDimensionalPlotPage* page() { return _page; } private: class TwoDimensionalPlotPage* _page; void updatePlotForSettings( const std::vector& kinds, const std::set& pols, const std::set& phases); double getValue(enum PhaseType Phase, const std::complex& val); void plotStatistic(QualityTablesFormatter::StatisticKind kind, SelectedPol pol, PhaseType phase, int lineIndex, const std::string& yDesc, bool second_axis); const StatisticsCollection* _statCollection; XYPlot _plot; }; #endif aoflagger-v3.5.1/aoqplot/controllers/histogrampagecontroller.cpp0000664000175000017500000002646614752462134023474 0ustar oleole#include "histogrampagecontroller.h" #include "../histogrampage.h" #include "../../quality/histogramtablesformatter.h" #include "../../quality/histogramcollection.h" #include "../../quality/rayleighfitter.h" #include "../../structures/msmetadata.h" #ifndef HAVE_EXP10 #define exp10(x) exp((2.3025850929940456840179914546844) * (x)) #endif HistogramPageController::HistogramPageController() : _page(nullptr), _histograms(nullptr), _summedPolarizationHistograms(nullptr) {} HistogramPageController::~HistogramPageController() { CloseStatistics(); } void HistogramPageController::readFromFile() { CloseStatistics(); HistogramTablesFormatter histogramTables(_statFilename); if (histogramTables.HistogramsExist()) { const MSMetaData set(_statFilename); const unsigned polarizationCount = set.PolarizationCount(); _histograms.reset(new HistogramCollection(polarizationCount)); _histograms->Load(histogramTables); } } void HistogramPageController::CloseStatistics() { _statFilename = std::string(); _histograms.reset(); _summedPolarizationHistograms.reset(); } void HistogramPageController::SetHistograms( const HistogramCollection* histograms) { CloseStatistics(); _histograms.reset(new HistogramCollection(*histograms)); _summedPolarizationHistograms.reset( _histograms->CreateSummedPolarizationCollection()); _histograms->CreateMissingBins(); _summedPolarizationHistograms->CreateMissingBins(); updatePlot(); } void HistogramPageController::updatePlot() { if (HasStatistics()) { _plot.Clear(); const unsigned polarizationCount = _histograms->PolarizationCount(); if (_drawXX) plotPolarization(*_histograms, 0); if (_drawXY && polarizationCount >= 2) plotPolarization(*_histograms, 1); if (_drawYX && polarizationCount >= 3) plotPolarization(*_histograms, 2); if (_drawYY && polarizationCount >= 4) plotPolarization(*_histograms, 3); if (_drawSum) plotPolarization(*_summedPolarizationHistograms, 0); if (_page != nullptr) _page->Redraw(); } } void HistogramPageController::plotPolarization( const HistogramCollection& histogramCollection, unsigned polarization) { LogHistogram totalHistogram, rfiHistogram; histogramCollection.GetTotalHistogramForCrossCorrelations(polarization, totalHistogram); histogramCollection.GetRFIHistogramForCrossCorrelations(polarization, rfiHistogram); plotPolarization(totalHistogram, rfiHistogram); } std::string HistogramPageController::SlopeText(const LogHistogram& histogram) { if (_automaticSlopeRange) { histogram.GetRFIRegion(_slopeStart, _slopeEnd); } std::stringstream str; const double slope = histogram.NormalizedSlope(_slopeStart, _slopeEnd), powerLawExp = histogram.PowerLawExponent(_slopeStart), powerLawExpError = histogram.PowerLawExponentStdError(_slopeStart, powerLawExp), offset = histogram.NormalizedSlopeOffset(_slopeStart, _slopeEnd, slope), error = histogram.NormalizedSlopeStdError(_slopeStart, _slopeEnd, slope), errorB = histogram.NormalizedSlopeStdDevBySampling( _slopeStart, _slopeEnd, slope, _deltaS), upperLimit = histogram.PowerLawUpperLimit(_slopeStart, slope, exp10(offset)), lowerLimit = histogram.PowerLawLowerLimit( _slopeStart, slope, exp10(offset), _slopeRFIRatio), lowerError = fabs(lowerLimit - histogram.PowerLawLowerLimit( _slopeStart, slope - error, exp10(offset), _slopeRFIRatio)), lowerLimit2 = histogram.PowerLawLowerLimit2( _slopeStart, slope, exp10(offset), _slopeRFIRatio); str << slope << "±" << error << "\n/±" << errorB << "\nb=" << exp10(offset) << "\nPL:" << powerLawExp << "±" << powerLawExpError << "\n[" << log10(lowerLimit) << "±" << lowerError << ';' << log10(upperLimit) << ']' << '\n' << log10(lowerLimit2); return str.str(); } void HistogramPageController::plotSlope(const LogHistogram& histogram, const std::string& title, bool useLowerLimit2) { double start, end; if (_automaticSlopeRange) { histogram.GetRFIRegion(start, end); } else { start = _slopeStart; end = _slopeEnd; } double xMin = log10(histogram.MinPositiveAmplitude()), rfiRatio = _slopeRFIRatio, slope = histogram.NormalizedSlope(start, end), offset = histogram.NormalizedSlopeOffset(start, end, slope), upperLimit = log10(histogram.PowerLawUpperLimit(start, slope, exp10(offset))), lowerLimit = useLowerLimit2 ? log10(histogram.PowerLawLowerLimit2( start, slope, exp10(offset), rfiRatio)) : log10(histogram.PowerLawLowerLimit( start, slope, exp10(offset), rfiRatio)); double xStart, xEnd; if (std::isfinite(lowerLimit)) xStart = lowerLimit; else xStart = log10(start) - 1.0; if (std::isfinite(upperLimit)) xEnd = upperLimit; else xEnd = log10(histogram.MaxAmplitude()); double yStart = xStart * slope + offset, yEnd = xEnd * slope + offset; _plot.StartLine(title, "Amplitude in arbitrary units (log)", "Frequency (log)"); if (useLowerLimit2 && std::isfinite(xMin)) _plot.PushDataPoint(xMin, yStart); _plot.PushDataPoint(xStart, yStart); _plot.PushDataPoint(xEnd, yEnd); } void HistogramPageController::plotPolarization( const LogHistogram& totalHistogram, const LogHistogram& rfiHistogram) { if (_totalHistogram) { _plot.StartLine("Total histogram", "Amplitude in arbitrary units (log)", "Frequency (log)"); addHistogramToPlot(totalHistogram); if (_fit || _subtractFit) { plotFit(totalHistogram, "Fit to total"); } if (_drawSlope) { plotSlope(totalHistogram, "Fitted slope", false); } if (_drawSlope2) { plotSlope(totalHistogram, "Fitted slope", true); } const std::string str = SlopeText(totalHistogram); _page->SetSlopeFrame(str); } if (_rfiHistogram) { _plot.StartLine("RFI histogram", "Amplitude in arbitrary units (log)", "Frequency (log)"); addHistogramToPlot(rfiHistogram); if (_fit || _subtractFit) { plotFit(rfiHistogram, "Fit to RFI"); } const std::string str = SlopeText(rfiHistogram); _page->SetSlopeFrame(str); if (_drawSlope) { plotSlope(rfiHistogram, "Fitted slope", false); } if (_drawSlope2) { plotSlope(rfiHistogram, "Fitted slope", true); } } if (_notRFIHistogram) { _plot.StartLine("Non-RFI histogram", "Amplitude in arbitrary units (log)", "Frequency (log)"); LogHistogram histogram(totalHistogram); histogram -= rfiHistogram; addHistogramToPlot(histogram); if (_fit || _subtractFit) { plotFit(histogram, "Fit to Non-RFI"); } } } void HistogramPageController::plotFit(const LogHistogram& histogram, const std::string& title) { double sigmaEstimate; sigmaEstimate = RayleighFitter::SigmaEstimate(histogram); if (_automaticFitRange) { RayleighFitter::FindFitRangeUnderRFIContamination( histogram.MinPositiveAmplitude(), sigmaEstimate, _fitStart, _fitEnd); } double sigma = RayleighFitter::SigmaEstimate(histogram, _fitStart, _fitEnd), n = RayleighFitter::NEstimate(histogram, _fitStart, _fitEnd); RayleighFitter fitter; fitter.SetFitLogarithmic(_fitLogarithmic); fitter.Fit(_fitStart, _fitEnd, histogram, sigma, n); if (_fit) { _plot.StartLine(title, "Amplitude in arbitrary units (log)", "Frequency (log)"); addRayleighToPlot(histogram, sigma, n); } if (_subtractFit) { _plot.StartLine(title, "Amplitude in arbitrary units (log)", "Frequency (log)"); addRayleighDifferenceToPlot(histogram, sigma, n); } std::stringstream str; str << "σ=1e" << log10(sigma) << ",n=1e" << log10(n) << '\n' << "n_t=1e" << log10(histogram.NormalizedTotalCount()) << '\n' << "mode=1e" << log10(histogram.AmplitudeWithMaxNormalizedCount()) << '\n' << "ε_R=" << RayleighFitter::ErrorOfFit(histogram, _fitStart, _fitEnd, sigma, n); _page->SetFitText(str.str()); } void HistogramPageController::addRayleighToPlot(const LogHistogram& histogram, double sigma, double n) { const bool derivative = _derivative; double x = histogram.MinPositiveAmplitude(); const double xend = sigma * 5.0; const double sigmaP2 = sigma * sigma; while (x < xend) { const double logx = log10(x); if (derivative) { const double dc = -(exp10(2.0 * x) - sigmaP2) / sigmaP2; if (std::isfinite(logx) && std::isfinite(dc)) _plot.PushDataPoint(logx, dc); } else { const double c = n * x / (sigmaP2)*exp(-x * x / (2 * sigmaP2)); const double logc = log10(c); if (std::isfinite(logx) && std::isfinite(logc)) _plot.PushDataPoint(logx, logc); } x *= 1.05; } } void HistogramPageController::addRayleighDifferenceToPlot( const LogHistogram& histogram, double sigma, double n) { const double sigmaP2 = sigma * sigma; const double minCount = histogram.MinPosNormalizedCount(); for (LogHistogram::iterator i = histogram.begin(); i != histogram.end(); ++i) { const double x = i.value(); const double c = n * x / (sigmaP2)*exp(-x * x / (2 * sigmaP2)); const double diff = fabs(i.normalizedCount() - c); if (diff >= minCount) { const double logx = log10(x); const double logc = log10(diff); if (std::isfinite(logx) && std::isfinite(logc)) _plot.PushDataPoint(logx, logc); } } } void HistogramPageController::addHistogramToPlot( const LogHistogram& histogram) { const bool derivative = _derivative; const bool staircase = _staircaseFunction; const bool normalize = _normalize; double deltaS = _deltaS; if (deltaS <= 1.0001) deltaS = 1.0001; for (LogHistogram::iterator i = histogram.begin(); i != histogram.end(); ++i) { double x = i.value(), logxStart, logxEnd; if (staircase) { logxStart = log10(i.binStart()); logxEnd = log10(i.binEnd()); } else { logxStart = log10(x); logxEnd = 0.0; // unused, but to avoid warning } if (derivative) { const double cslope = histogram.NormalizedSlope(x / deltaS, x * deltaS); // if(std::isfinite(logxStart) && std::isfinite(cslope)) _plot.PushDataPoint(logxStart, cslope); if (staircase) // && std::isfinite(logxEnd) && std::isfinite(cslope)) _plot.PushDataPoint(logxEnd, cslope); } else { const double logc = log10(normalize ? i.normalizedCount() : i.unnormalizedCount()); // if(std::isfinite(logxStart) && std::isfinite(logc)) _plot.PushDataPoint(logxStart, logc); if (staircase) // && std::isfinite(logxEnd) && std::isfinite(logc)) _plot.PushDataPoint(logxEnd, logc); } } } aoflagger-v3.5.1/aoqplot/controllers/frequencypagecontroller.h0000664000175000017500000000567614752462134023145 0ustar oleole#ifndef FREQUENCY_PAGE_CONTROLLER_H #define FREQUENCY_PAGE_CONTROLLER_H #include #include #include #include #include "aoqplotpagecontroller.h" class FrequencyPageController final : public AOQPlotPageController { public: void SetPerformFT(bool performFT) { _performFT = performFT; } protected: void processStatistics(const StatisticsCollection* statCollection, const std::vector&) override { _statistics.clear(); const std::map& map = statCollection->FrequencyStatistics(); for (std::map::const_iterator i = map.begin(); i != map.end(); ++i) { _statistics.insert(std::pair( i->first / 1000000.0, i->second)); } } const std::map& getStatistics() const override { return _statistics; } void startLine(XYPlot& plot, const std::string& name, int lineIndex, const std::string& yAxisDesc, bool second_axis) override { XYPointSet* points; if (_performFT) points = &plot.StartLine(name, "Time (μs)", yAxisDesc); else points = &plot.StartLine(name, "Frequency (MHz)", yAxisDesc); points->SetUseSecondYAxis(second_axis); } void processPlot(XYPlot& plot) override { if (_performFT) { performFt(plot); } } void performFt(XYPlot& plot) { size_t count = plot.PointSetCount(); for (size_t line = 0; line < count; ++line) { XYPointSet& pointSet = plot.GetPointSet(line); std::vector>> output; const double min = pointSet.MinX(); const double width = pointSet.MaxX() - min; const double fStart = -2.0 * M_PI * (double)pointSet.Size() / width; const double fEnd = 2.0 * M_PI * (double)pointSet.Size() / width; const double fStep = (fEnd - fStart) / (double)pointSet.Size(); for (double f = fStart; f < fEnd; f += fStep) { std::pair> newElement( f / (2.0 * M_PI), std::complex(0.0, 0.0)); std::complex& nextStat = newElement.second; for (size_t i = 0; i != pointSet.Size(); ++i) { const double t_f = pointSet.GetX(i) * f; const double val = pointSet.GetY(i); if (std::isfinite(val)) nextStat += std::complex(val * cos(t_f), val * sin(t_f)); } output.push_back(newElement); } pointSet.Clear(); for (std::vector>>::const_iterator i = output.begin(); i != output.end(); ++i) { double real = i->second.real(), imag = i->second.imag(); pointSet.PushDataPoint(i->first, sqrt(real * real + imag * imag)); } } } private: std::map _statistics; bool _performFT = false; }; #endif aoflagger-v3.5.1/aoqplot/controllers/histogrampagecontroller.h0000664000175000017500000000755614752462134023140 0ustar oleole#ifndef HISTOGRAM_PAGE_CONTROLLER_H #define HISTOGRAM_PAGE_CONTROLLER_H #include #include #include "aoqplotpagecontroller.h" #include "../../plot/xyplot.h" class HistogramPageController : public AOQPageController { public: HistogramPageController(); ~HistogramPageController(); void Attach(class HistogramPage* page) { _page = page; } void SetHistograms(const HistogramCollection* histograms) override final; void SetHistograms(const std::string& filename) { _statFilename = filename; readFromFile(); updatePlot(); } void CloseStatistics() override final; bool HasStatistics() const { return _histograms != nullptr; } XYPlot& Plot() { return _plot; } std::string SlopeText(const class LogHistogram& histogram); void SetAutomaticFitRange(bool automaticFitRange) { _automaticFitRange = automaticFitRange; } void SetFitStart(double fitStart) { _fitStart = fitStart; } void SetFitEnd(double fitEnd) { _fitEnd = fitEnd; } void SetFitLogarithmic(bool fitLog) { _fitLogarithmic = fitLog; } double FitStart() { return _fitStart; } double FitEnd() { return _fitEnd; } void SetAutomaticSlopeRange(bool automaticSlopeRange) { _automaticSlopeRange = automaticSlopeRange; } void SetSlopeStart(double start) { _slopeStart = start; } void SetSlopeEnd(double end) { _slopeEnd = end; } void SetSlopeRFIRatio(double rfiRatio) { _slopeRFIRatio = rfiRatio; } double SlopeStart() { return _slopeStart; } double SlopeEnd() { return _slopeEnd; } void SetDrawTotal(bool drawTotal) { _totalHistogram = drawTotal; } void SetDrawFit(bool drawFit) { _fit = drawFit; } void SetDrawSubtractedFit(bool drawSubtrFit) { _subtractFit = drawSubtrFit; } void SetDrawSlope(bool drawSlope) { _drawSlope = drawSlope; } void SetDrawSlope2(bool drawSlope2) { _drawSlope2 = drawSlope2; } void SetDrawRFI(bool drawRFI) { _rfiHistogram = drawRFI; } void SetDrawNonRFI(bool drawNonRFI) { _notRFIHistogram = drawNonRFI; } void SetDerivative(bool derivative) { _derivative = derivative; } void SetStaircase(bool staircaseFunction) { _staircaseFunction = staircaseFunction; } void SetNormalize(bool normalize) { _normalize = normalize; } void SetDeltaS(double deltaS) { _deltaS = deltaS; } void SetDrawXX(bool drawXX) { _drawXX = drawXX; } void SetDrawXY(bool drawXY) { _drawXY = drawXY; } void SetDrawYX(bool drawYX) { _drawYX = drawYX; } void SetDrawYY(bool drawYY) { _drawYY = drawYY; } void SetDrawSum(bool drawSum) { _drawSum = drawSum; } private: void updatePlot(); void addHistogramToPlot(const class LogHistogram& histogram); void addRayleighToPlot(const class LogHistogram& histogram, double sigma, double n); void addRayleighDifferenceToPlot(const LogHistogram& histogram, double sigma, double n); void plotPolarization(const HistogramCollection& histogramCollection, unsigned polarization); void plotPolarization(const class LogHistogram& totalHistogram, const class LogHistogram& rfiHistogram); void plotFit(const class LogHistogram& histogram, const std::string& title); void plotSlope(const class LogHistogram& histogram, const std::string& title, bool useLowerLimit2); void readFromFile(); class HistogramPage* _page; std::string _statFilename; XYPlot _plot; std::unique_ptr _histograms; std::unique_ptr _summedPolarizationHistograms; bool _drawXX, _drawXY, _drawYX, _drawYY, _drawSum; bool _automaticFitRange; double _fitStart, _fitEnd; bool _fitLogarithmic, _automaticSlopeRange; double _slopeStart, _slopeEnd, _slopeRFIRatio; bool _totalHistogram, _fit, _subtractFit, _drawSlope, _drawSlope2, _rfiHistogram, _notRFIHistogram; bool _derivative, _staircaseFunction, _normalize; double _deltaS; }; #endif aoflagger-v3.5.1/aoqplot/controllers/aoqplotpagecontroller.cpp0000664000175000017500000001147414752462134023147 0ustar oleole#include "aoqplotpagecontroller.h" #include "../twodimensionalplotpage.h" #include "../../quality/statisticsderivator.h" #include "../../util/logger.h" AOQPlotPageController::AOQPlotPageController() : _page(nullptr), _statCollection(nullptr) {} void AOQPlotPageController::UpdatePlot() { if (_page != nullptr) { updatePlotForSettings(_page->GetSelectedKinds(), _page->GetSelectedPolarizations(), _page->GetSelectedPhases()); } } void AOQPlotPageController::updatePlotForSettings( const std::vector& kinds, const std::set& pols, const std::set& phases) { if (HasStatistics()) { _plot.Clear(); std::map units; for (const QualityTablesFormatter::StatisticKind k : kinds) { const std::string unit_str = StatisticsDerivator::GetUnits(k); std::map::const_iterator iterator = units.find(unit_str); if (iterator == units.end()) { units.emplace(unit_str, units.size() % 2); } } _plot.Y2Axis().SetShow(false); int index = 0; for (const QualityTablesFormatter::StatisticKind k : kinds) { const std::string unit_str = StatisticsDerivator::GetUnits(k); const int axis_number = units.find(unit_str)->second; const bool multiple_units = units.size() > 3 || (units.size() > 2 && (axis_number % 2) == 0); XYPlotAxis& y_axis = axis_number == 1 ? _plot.Y2Axis() : _plot.YAxis(); y_axis.SetShow(true); for (const SelectedPol p : pols) { for (const PhaseType ph : phases) { const std::string description = multiple_units ? "Value" : StatisticsDerivator::GetDescWithUnits(k); plotStatistic(k, p, ph, index, description, axis_number == 1); ++index; } } } processPlot(_plot); if (_page != nullptr) { _page->Redraw(); } } } double AOQPlotPageController::getValue(enum PhaseType phase, const std::complex& val) { switch (phase) { default: case AmplitudePhaseType: return sqrt(val.real() * val.real() + val.imag() * val.imag()); case PhasePhaseType: return atan2(val.imag(), val.real()); case RealPhaseType: return val.real(); case ImaginaryPhaseType: return val.imag(); } } void AOQPlotPageController::plotStatistic( QualityTablesFormatter::StatisticKind kind, SelectedPol pol, PhaseType phase, int lineIndex, const std::string& yDesc, bool second_axis) { const StatisticsDerivator derivator(*_statCollection); const size_t polCount = _statCollection->PolarizationCount(); const std::map& statistics = getStatistics(); std::ostringstream s; int polIndex = -1; s << StatisticsDerivator::GetDescription(kind); switch (pol) { case PolI: s << ", pol I"; polIndex = 0; break; case PolPP: s << ", pol PP"; polIndex = 0; break; case PolPQ: s << ", pol PQ"; if (polCount == 4) polIndex = 1; break; case PolQP: s << ", pol QP"; if (polCount == 4) polIndex = 2; break; case PolQQ: s << ", pol QQ"; if (polCount == 4) polIndex = 3; else if (polCount == 2) polIndex = 1; break; } if (phase == RealPhaseType) s << " (real)"; else if (phase == ImaginaryPhaseType) s << " (imag)"; if (polIndex >= 0) { startLine(_plot, s.str(), lineIndex, yDesc, second_axis); for (std::map::const_iterator i = statistics.begin(); i != statistics.end(); ++i) { const double x = i->first; std::complex val; if (pol == PolI) { const std::complex valA = derivator.GetComplexStatistic(kind, i->second, 0); const std::complex valB = derivator.GetComplexStatistic(kind, i->second, polCount - 1); val = valA * 0.5l + valB * 0.5l; } else { val = derivator.GetComplexStatistic(kind, i->second, polIndex); } _plot.PushDataPoint(x, getValue(phase, val)); } } } void AOQPlotPageController::SavePdf( const string& filename, QualityTablesFormatter::StatisticKind kind) { const std::vector kinds{kind}; const std::set pols{PolI}; const std::set phases{AmplitudePhaseType}; updatePlotForSettings(kinds, pols, phases); _plot.SavePdf(filename, 640, 480); } void AOQPlotPageController::SetStatistics( const StatisticsCollection* statCollection, const std::vector& antennas) { processStatistics(statCollection, antennas); _statCollection = statCollection; UpdatePlot(); } aoflagger-v3.5.1/aoqplot/controllers/aoqpagecontroller.h0000664000175000017500000000073214752462134021710 0ustar oleole#ifndef AOQ_PAGE_CONTROLLER_H #define AOQ_PAGE_CONTROLLER_H #include "../../structures/antennainfo.h" #include class AOQPageController { public: virtual ~AOQPageController() {} virtual void SetStatistics(const class StatisticsCollection* statCollection, const std::vector& antennas) {} virtual void SetHistograms(const class HistogramCollection* histograms) {} virtual void CloseStatistics() = 0; }; #endif aoflagger-v3.5.1/aoqplot/controllers/tfpagecontroller.h0000664000175000017500000000142114752462134021535 0ustar oleole#ifndef TF_PAGE_CONTROLLER_H #define TF_PAGE_CONTROLLER_H #include #include #include "heatmappagecontroller.h" class TFPageController : public HeatMapPageController { public: TFPageController(); void SetStatistics(const StatisticsCollection* statCollection, const std::vector&) override final { _statCollection = statCollection; UpdateImage(); } void CloseStatistics() override final { _statCollection = nullptr; } bool HasStatistics() const { return _statCollection != nullptr; } protected: std::pair constructImage( QualityTablesFormatter::StatisticKind kind) final override; private: const StatisticsCollection* _statCollection; }; #endif aoflagger-v3.5.1/aoqplot/controllers/timepagecontroller.h0000664000175000017500000000147314752462134022071 0ustar oleole#ifndef TIME_PAGE_CONTROLLER_H #define TIME_PAGE_CONTROLLER_H #include #include #include "aoqplotpagecontroller.h" #include "../../quality/statisticscollection.h" class TimePageController final : public AOQPlotPageController { protected: const std::map& getStatistics() const override { if (getStatCollection()->AllTimeStatistics().empty()) { static std::map empty_set; return empty_set; } else { return getStatCollection()->TimeStatistics(); } } void startLine(XYPlot& plot, const std::string& name, int lineIndex, const std::string& yAxisDesc, bool second_axis) override { XYPointSet& points = plot.StartLine(name, "Time", yAxisDesc); points.SetUseSecondYAxis(second_axis); } }; #endif aoflagger-v3.5.1/aoqplot/controllers/tfpagecontroller.cpp0000664000175000017500000000206214752462134022072 0ustar oleole#include "tfpagecontroller.h" #include "../../quality/statisticscollection.h" #include "../../quality/statisticsderivator.h" TFPageController::TFPageController() : _statCollection(nullptr) { HeatMap& map = Plot(); map.SetXAxisDescription("Time index"); map.SetXAxisType(AxisType::kTime); map.SetYAxisDescription("Frequency index"); } std::pair TFPageController::constructImage(QualityTablesFormatter::StatisticKind kind) { if (HasStatistics()) { StatisticsDerivator derivator(*_statCollection); std::pair data = derivator.CreateTFData(kind); if (data.second == nullptr) { Plot().SetXAxisDescription("Time index"); Plot().SetYAxisDescription("Frequency index"); } else { Plot().SetXAxisDescription("Time"); Plot().SetYAxisDescription("Frequency (MHz)"); } return data; } else { return std::pair( TimeFrequencyData(), TimeFrequencyMetaDataCPtr()); } } aoflagger-v3.5.1/aoqplot/controllers/heatmappagecontroller.h0000664000175000017500000000436414752462134022554 0ustar oleole#ifndef HEATMAP_PAGE_CONTROLLER_H #define HEATMAP_PAGE_CONTROLLER_H #include #include #include "aoqplotpagecontroller.h" #include "../../rfigui/maskedheatmap.h" #include "../../structures/timefrequencydata.h" #include class HeatMapPageController : public AOQPageController { public: HeatMapPageController(); void Attach(class GrayScalePlotPage* page) { _page = page; } void SavePdf(const std::string& filename, QualityTablesFormatter::StatisticKind kind, unsigned width, unsigned height) { updateImageImpl(kind, aocommon::Polarization::StokesI, TimeFrequencyData::AmplitudePart); _heatMap.SavePdf(filename, width, height); } HeatMap& Plot() { return _heatMap; } void UpdateImage() { updateImageImpl(_statisticKind, _polarization, _phase); } void SetKind(QualityTablesFormatter::StatisticKind statisticKind) { _statisticKind = statisticKind; } void SetPolarization(aocommon::PolarizationEnum polarization) { _polarization = polarization; } void SetPhase(enum TimeFrequencyData::ComplexRepresentation phase) { _phase = phase; } enum Normalization { Mean, Winsorized, Median }; void SetNormalization(Normalization normalization) { _normalization = normalization; } protected: virtual std::pair constructImage(QualityTablesFormatter::StatisticKind kind) = 0; private: Image2D normalizeXAxis(const Image2D& input); Image2D normalizeYAxis(const Image2D& input); void updateImageImpl(QualityTablesFormatter::StatisticKind statisticKind, aocommon::PolarizationEnum polarisation, enum TimeFrequencyData::ComplexRepresentation phase); void setToPolarization(TimeFrequencyData& data, aocommon::PolarizationEnum polarisation); void setToPhase(TimeFrequencyData& data, enum TimeFrequencyData::ComplexRepresentation phase); MaskedHeatMap _heatMap; class GrayScalePlotPage* _page; QualityTablesFormatter::StatisticKind _statisticKind; aocommon::PolarizationEnum _polarization; enum TimeFrequencyData::ComplexRepresentation _phase; enum Normalization _normalization; }; #endif aoflagger-v3.5.1/aoqplot/baselineplotpage.cpp0000664000175000017500000000255115002644503017462 0ustar oleole#include #include "baselineplotpage.h" #include "controllers/baselinepagecontroller.h" BaselinePlotPage::BaselinePlotPage(BaselinePageController* controller) : GrayScalePlotPage(controller), _controller(controller) { grayScaleWidget().OnMouseMovedEvent().connect( sigc::mem_fun(*this, &BaselinePlotPage::onMouseMoved)); } BaselinePlotPage::~BaselinePlotPage() = default; void BaselinePlotPage::onMouseMoved(double x, double y) { const MaskedHeatMap& map = static_cast(grayScaleWidget().Plot()); size_t image_x; size_t image_y; if (map.UnitToImage(x, y, image_x, image_y)) { const std::string antenna1Name = _controller->AntennaName(x); const std::string antenna2Name = _controller->AntennaName(y); const QualityTablesFormatter::StatisticKind kind = getSelectedStatisticKind(); const std::string& kindName = QualityTablesFormatter::KindToName(kind); std::stringstream text; const size_t stride = static_cast(grayScaleWidget().Plot()).Image().Stride(); text << "Correlation " << antenna1Name << " (" << image_x << ") x " << antenna2Name << " (" << image_y << "), " << kindName << " = " << static_cast(grayScaleWidget().Plot()) .Image() .Data()[image_y * stride + image_x]; _signalStatusChange(text.str()); } } aoflagger-v3.5.1/aoqplot/timefrequencyplotpage.h0000664000175000017500000000050214752462134020230 0ustar oleole#ifndef GUI_QUALITY__TIMEFREQUENCYPLOTPAGE_H #define GUI_QUALITY__TIMEFREQUENCYPLOTPAGE_H #include "grayscaleplotpage.h" class TimeFrequencyPlotPage : public GrayScalePlotPage { public: explicit TimeFrequencyPlotPage(class TFPageController* controller); private: void onMouseMoved(double x, double y); }; #endif aoflagger-v3.5.1/aoqplot/summarypage.h0000664000175000017500000001417615106657110016154 0ustar oleole#ifndef GUI_QUALITY__SUMMARYPAGE_H #define GUI_QUALITY__SUMMARYPAGE_H #include #include #include #include #include #include #include "../quality/statisticscollection.h" #include "../quality/statisticsderivator.h" #include "plotsheet.h" #include #ifndef HAVE_EXP10 #define exp10(x) exp((2.3025850929940456840179914546844) * (x)) #endif class SummaryPageController : public AOQPageController { public: void Attach(class SummaryPage* page) { _page = page; } void SetStatistics(const StatisticsCollection* statCollection, const std::vector&) final override; void CloseStatistics() final override; bool HasStatistics() const { return _statCollection != nullptr; } private: void updateText(); void addText(std::ostringstream& str, DefaultStatistics& statistics) { unsigned long totalRFICount = 0; unsigned long totalCount = 0; const unsigned polarizationCount = statistics.PolarizationCount(); std::vector stdDev(polarizationCount), dStdDev(polarizationCount), variance(polarizationCount), dVariance(polarizationCount); for (unsigned p = 0; p < polarizationCount; ++p) { totalRFICount += statistics.rfiCount[p]; totalCount += statistics.rfiCount[p] + statistics.count[p]; variance[p] = StatisticsDerivator::VarianceAmplitude( statistics.count[p], statistics.sum[p], statistics.sumP2[p]); dVariance[p] = StatisticsDerivator::VarianceAmplitude( statistics.dCount[p], statistics.dSum[p], statistics.dSumP2[p]); stdDev[p] = StatisticsDerivator::StandardDeviationAmplitude( statistics.count[p], statistics.sum[p], statistics.sumP2[p]); dStdDev[p] = StatisticsDerivator::StandardDeviationAmplitude( statistics.dCount[p], statistics.dSum[p], statistics.dSumP2[p]); } double rfiRatioValue = round(((double)totalRFICount * 10000.0 / (double)totalCount)) * 0.01; double countExp = floor(log10(totalCount)); double countMantissa = totalCount / exp10(countExp); str << "Sample count = " << round(countMantissa * 100.0) * 0.01 << " x 10^" << countExp << "\n"; str << "Total RFI ratio = " << rfiRatioValue << "%\n"; str << "Standard deviation amplitude = "; addValues(&stdDev[0], polarizationCount, str); str << " Jy\nDifferential stddev amplitude = "; addValues(&dStdDev[0], polarizationCount, str); str << " Jy\nVariance amplitude = "; addValues(&variance[0], polarizationCount, str); str << " Jy\nDifferential variance amplitude = "; addValues(&dVariance[0], polarizationCount, str); str << " Jy\n"; } void addBaselineAverages(std::ostringstream& str) { const BaselineStatisticsMap& map = _statCollection->BaselineStatistics(); std::vector> list = map.BaselineList(); std::vector totalStdDev(map.PolarizationCount()), totalSNR(map.PolarizationCount()); std::vector count(map.PolarizationCount()); for (size_t p = 0; p < map.PolarizationCount(); ++p) { totalStdDev[p] = 0.0; totalSNR[p] = 0.0; count[p] = 0; } for (std::vector>::const_iterator i = list.begin(); i != list.end(); ++i) { unsigned a1 = i->first, a2 = i->second; if (a1 != a2) { const DefaultStatistics& stat = map.GetStatistics(a1, a2); for (size_t p = 0; p < map.PolarizationCount(); ++p) { const double thisStdDev = StatisticsDerivator::GetStatisticAmplitude( QualityTablesFormatter::StandardDeviationStatistic, stat, p), thisSNR = StatisticsDerivator::GetStatisticAmplitude( QualityTablesFormatter::SignalToNoiseStatistic, stat, p); if (std::isfinite(thisStdDev) && std::isfinite(thisSNR)) { totalStdDev[p] += thisStdDev; totalSNR[p] += thisSNR; ++count[p]; } } } } for (size_t p = 0; p < map.PolarizationCount(); ++p) { totalStdDev[p] /= (double)count[p]; totalSNR[p] /= (double)count[p]; } str << "Average standard deviation = "; addValues(&totalStdDev[0], map.PolarizationCount(), str); str << " Jy\nAverage signal to noise ratio = "; addValues(&totalSNR[0], map.PolarizationCount(), str); str << " Jy\n(calculated with BaselineMean/BaselineDStdDev)\n"; } void addValues(const double* values, unsigned polarizationCount, std::ostringstream& s) { s << '[' << values[0]; for (unsigned p = 1; p < polarizationCount; ++p) { s << ", " << values[p]; } s << ']'; } const StatisticsCollection* _statCollection; class SummaryPage* _page; }; class SummaryPage : public PlotSheet { public: explicit SummaryPage(SummaryPageController* controller) : _controller(controller) { append(_textView); _textView.set_expand(true); _controller->Attach(this); } void SetText(const std::string& str) { Glib::RefPtr buffer = _textView.get_buffer(); buffer->set_text(str); } private: Gtk::TextView _textView; SummaryPageController* _controller; }; inline void SummaryPageController::SetStatistics( const StatisticsCollection* statCollection, const std::vector&) { _statCollection = statCollection; updateText(); } inline void SummaryPageController::CloseStatistics() { _statCollection = nullptr; _page->SetText("No open measurement set"); } inline void SummaryPageController::updateText() { DefaultStatistics statistics(_statCollection->PolarizationCount()); std::ostringstream str; _statCollection->GetGlobalCrossBaselineStatistics(statistics); str << "Statistics of cross-correlated baselines\n"; addText(str, statistics); str << "\nAverages over cross-correlated baselines\n"; addBaselineAverages(str); _statCollection->GetGlobalAutoBaselineStatistics(statistics); str << "\nStatistics of auto-correlated baselines\n"; addText(str, statistics); _page->SetText(str.str()); } #endif aoflagger-v3.5.1/aoqplot/aoqplotwindow.cpp0000664000175000017500000001464115132201277017057 0ustar oleole#include "aoqplotwindow.h" #include "controllers/antennapagecontroller.h" #include "controllers/aoqplotcontroller.h" #include "controllers/baselinepagecontroller.h" #include "controllers/blengthpagecontroller.h" #include "controllers/frequencypagecontroller.h" #include "controllers/histogrampagecontroller.h" #include "controllers/tfpagecontroller.h" #include "controllers/timepagecontroller.h" #include #include #include #include "../structures/msmetadata.h" #include "../quality/histogramtablesformatter.h" #include "../quality/histogramcollection.h" #include "../quality/statisticscollection.h" #include "baselineplotpage.h" #include "blengthplotpage.h" #include "frequencyplotpage.h" #include "histogrampage.h" #include "summarypage.h" #include "timefrequencyplotpage.h" AOQPlotWindow::AOQPlotWindow(AOQPlotController* controller) : _controller(controller), _activeSheetIndex(-1), page_menu_button_() { set_default_icon_name("aoqplot"); page_menu_button_.set_icon_name("open-menu-symbolic"); std::shared_ptr page_menu = Gio::Menu::create(); page_menu->append("Baselines", "win.page_selection(0)"); page_menu->append("Antennae", "win.page_selection(1)"); page_menu->append("Baseline lengths", "win.page_selection(2)"); page_menu->append("Time", "win.page_selection(3)"); page_menu->append("Frequency", "win.page_selection(4)"); page_menu->append("Time-frequency", "win.page_selection(5)"); page_menu->append("Summary", "win.page_selection(6)"); page_menu->append("Histograms", "win.page_selection(7)"); auto actions = Gio::SimpleActionGroup::create(); page_selection_ = Gio::SimpleAction::create_radio_integer("page_selection", 0); page_selection_->signal_change_state().connect( [this](const Glib::VariantBase& state) { const int value = static_cast&>(state).get(); onChangeSheet(value); }); actions->add_action(page_selection_); insert_action_group("win", actions); page_menu_button_.set_menu_model(page_menu); toolbar_.append(page_menu_button_); _vBox.append(toolbar_); _vBox.append(_statusBar); _statusBar.push( "Quality plot util is ready. Author: André Offringa " "(offringa@gmail.com)"); _vBox.set_expand(true); set_child(_vBox); _openOptionsWindow.SignalOpen().connect( sigc::mem_fun(*this, &AOQPlotWindow::onOpenOptionsSelected)); _controller->Attach(this); } void AOQPlotWindow::Open(const std::vector& files) { _openOptionsWindow.ShowForFile(files); } void AOQPlotWindow::onOpenOptionsSelected(const std::vector& files, bool downsampleTime, bool downsampleFreq, size_t timeCount, size_t freqCount, bool correctHistograms) { _controller->ReadStatistics(files, downsampleTime, downsampleFreq, timeCount, freqCount, correctHistograms); _activeSheetIndex = -1; onChangeSheet(0); show(); } void AOQPlotWindow::onStatusChange(const std::string& newStatus) { _statusBar.pop(); _statusBar.push(newStatus); } void AOQPlotWindow::onChangeSheet(int selectedSheet) { if (selectedSheet != _activeSheetIndex) { if (plot_page_) _vBox.remove(*plot_page_); // In gtk 4, toggle buttons that are part of a group need to be disconnected // from the group before they are removed. auto* child = toolbar_.get_last_child(); while (child != &page_menu_button_) { if (Gtk::ToggleButton* toggle = dynamic_cast(child); toggle) { toggle->unset_group(); } toolbar_.remove(*child); child = toolbar_.get_last_child(); } page_selection_->set_state(Glib::Variant::create(selectedSheet)); switch (selectedSheet) { case 0: _pageController.reset(new BaselinePageController()); plot_page_.reset(new BaselinePlotPage( static_cast(_pageController.get()))); SetStatus("Baseline statistics"); break; case 1: _pageController.reset(new AntennaePageController()); plot_page_.reset(new TwoDimensionalPlotPage( static_cast(_pageController.get()))); SetStatus("Antennae statistics"); break; case 2: _pageController.reset(new BLengthPageController()); plot_page_.reset(new BLengthPlotPage( static_cast(_pageController.get()))); SetStatus("Baseline length statistics"); break; case 3: _pageController.reset(new TimePageController()); plot_page_.reset(new TwoDimensionalPlotPage( static_cast(_pageController.get()))); SetStatus("Time statistics"); break; case 4: _pageController.reset(new FrequencyPageController()); plot_page_.reset(new FrequencyPlotPage( static_cast(_pageController.get()))); SetStatus("Frequency statistics"); break; case 5: _pageController.reset(new TFPageController()); plot_page_.reset(new TimeFrequencyPlotPage( static_cast(_pageController.get()))); SetStatus("Time-frequency statistics"); break; case 6: _pageController.reset(new SummaryPageController()); plot_page_.reset(new SummaryPage( static_cast(_pageController.get()))); SetStatus("Summary"); break; case 7: _pageController.reset(new HistogramPageController()); plot_page_.reset(new HistogramPage( static_cast(_pageController.get()))); SetStatus("Histograms"); break; } _activeSheetIndex = selectedSheet; _controller->Initialize(_pageController.get(), selectedSheet != 5); plot_page_->SignalStatusChange().connect( sigc::mem_fun(*this, &AOQPlotWindow::onStatusChange)); plot_page_->InitializeToolbar(toolbar_); plot_page_->set_expand(true); _vBox.insert_child_after(*plot_page_, toolbar_); } } void AOQPlotWindow::ShowError(const std::string& message) { dialog_ = std::make_unique(*this, message, false, Gtk::MessageType::ERROR); dialog_->show(); } aoflagger-v3.5.1/aoqplot/histogrampage.h0000664000175000017500000000550015105046727016450 0ustar oleole#ifndef GUI_QUALITY__HISTOGRAMPAGE_H #define GUI_QUALITY__HISTOGRAMPAGE_H #include #include #include #include #include #include #include #include #include #include "../plot/plotwidget.h" #include "plotsheet.h" class HistogramPage : public PlotSheet { public: explicit HistogramPage(class HistogramPageController* controller); ~HistogramPage(); void updatePlot(); void Redraw() { _plotWidget.Update(); } void SetSlopeFrame(const std::string& str); void SetFitText(const std::string& str) { _fitTextView.get_buffer()->set_text(str); } private: void onPlotPropertiesClicked(); void onDataExportClicked(); void updateSlopeFrame(const class LogHistogram& histogram); std::string SlopeText(std::stringstream& str, const LogHistogram& histogram, bool updateRange); void updateDataWindow(); void onAutoRangeClicked() { bool autoRange = _fitAutoRangeButton.get_active(); _fitStartEntry.set_sensitive(!autoRange); _fitEndEntry.set_sensitive(!autoRange); if (autoRange) updatePlot(); } void onSlopeAutoRangeClicked() { bool autoRange = _slopeAutoRangeButton.get_active(); _slopeStartEntry.set_sensitive(!autoRange); _slopeEndEntry.set_sensitive(!autoRange); if (autoRange) updatePlot(); } class HistogramPageController* _controller; Gtk::Expander _expander; Gtk::Box _sideBox{Gtk::Orientation::VERTICAL}; Gtk::Frame _histogramTypeFrame; Gtk::Box _histogramTypeBox{Gtk::Orientation::VERTICAL}; Gtk::CheckButton _totalHistogramButton, _rfiHistogramButton, _notRFIHistogramButton; Gtk::Frame _polarizationFrame; Gtk::Box _polarizationBox{Gtk::Orientation::VERTICAL}; Gtk::CheckButton _xxPolarizationButton, _xyPolarizationButton, _yxPolarizationButton, _yyPolarizationButton, _sumPolarizationButton; Gtk::Frame _fitFrame; Gtk::Box _fitBox{Gtk::Orientation::VERTICAL}; Gtk::CheckButton _fitButton, _subtractFitButton, _fitLogarithmicButton, _fitAutoRangeButton; Gtk::Entry _fitStartEntry, _fitEndEntry; Gtk::TextView _fitTextView; Gtk::Frame _functionFrame; Gtk::Box _functionBox{Gtk::Orientation::VERTICAL}; Gtk::CheckButton _nsButton, _dndsButton; Gtk::Entry _deltaSEntry; Gtk::CheckButton _staircaseFunctionButton, _normalizeButton; Gtk::Button _plotPropertiesButton, _dataExportButton; Gtk::Frame _slopeFrame; Gtk::Box _slopeBox{Gtk::Orientation::VERTICAL}; Gtk::TextView _slopeTextView; Gtk::CheckButton _drawSlopeButton, _drawSlope2Button; Gtk::CheckButton _slopeAutoRangeButton; Gtk::Entry _slopeStartEntry, _slopeEndEntry, _slopeRFIRatio; PlotWidget _plotWidget; class PlotPropertiesWindow* _plotPropertiesWindow; class DataWindow* _dataWindow; }; #endif aoflagger-v3.5.1/aoluarunner/0000775000175000017500000000000015146315735014324 5ustar oleoleaoflagger-v3.5.1/aoluarunner/baselineiterator.h0000664000175000017500000000553014752462134020032 0ustar oleole#ifndef RFISTRATEGYFOREACHBASELINEACTION_H #define RFISTRATEGYFOREACHBASELINEACTION_H #include "options.h" #include "../imagesets/imageset.h" #include #include #include #include #include #include #include #include class BaselineIterator { public: BaselineIterator(std::mutex* ioMutex, const Options& options); ~BaselineIterator(); void Run(imagesets::ImageSet& imageSet, class LuaThreadGroup& lua, class ScriptData& scriptData); private: bool IsSequenceSelected(imagesets::ImageSetIndex& index); imagesets::ImageSetIndex GetNextIndex(); static std::string memToStr(double memSize); void SetExceptionOccured(); void SetFinishedBaselines(); // void SetProgress(ProgressListener &progress, int no, int count, const // std::string& taskName, int threadId); size_t BaselineProgress() { std::lock_guard lock(_mutex); return _baselineProgress; } void IncBaselineProgress() { std::lock_guard lock(_mutex); ++_baselineProgress; } void WaitForReadBufferAvailable(size_t maxSize) { std::unique_lock lock(_mutex); while (_baselineBuffer.size() > maxSize && !_exceptionOccured) _dataProcessed.wait(lock); } std::unique_ptr GetNextBaseline() { std::unique_lock lock(_mutex); while (_baselineBuffer.size() == 0 && !_exceptionOccured && !_finishedBaselines) _dataAvailable.wait(lock); if ((_finishedBaselines && _baselineBuffer.size() == 0) || _exceptionOccured) { return nullptr; } else { std::unique_ptr next = std::move(_baselineBuffer.top()); _baselineBuffer.pop(); _dataProcessed.notify_one(); return next; } } size_t GetBaselinesInBufferCount() { std::lock_guard lock(_mutex); return _baselineBuffer.size(); } struct ProcessingThread { ProcessingThread(BaselineIterator& parent, size_t threadIndex) : _parent(parent), _threadIndex(threadIndex) {} BaselineIterator& _parent; size_t _threadIndex; void operator()(); }; struct ReaderThread { explicit ReaderThread(BaselineIterator& parent) : _parent(parent) {} void operator()(); BaselineIterator& _parent; }; const Options& _options; LuaThreadGroup* _lua; imagesets::ImageSet* _imageSet; size_t _sequenceCount, _nextIndex; size_t _threadCount; imagesets::ImageSetIndex _loopIndex; std::unique_ptr _writeThread; std::mutex _mutex, *_ioMutex; std::condition_variable _dataAvailable, _dataProcessed; std::stack> _baselineBuffer; bool _finishedBaselines; bool _exceptionOccured; size_t _baselineProgress; class ScriptData* _globalScriptData; }; #endif aoflagger-v3.5.1/aoluarunner/runner.cpp0000664000175000017500000003771014752462134016347 0ustar oleole#include "runner.h" #include "baselineiterator.h" #include "../lua/luathreadgroup.h" #include "../lua/scriptdata.h" #include "../lua/optionsfunction.h" #include "../lua/telescopefile.h" #include "../quality/statisticscollection.h" #include "../structures/msmetadata.h" #include "../imagesets/h5imageset.h" #include "../imagesets/joinedspwset.h" #include "../imagesets/msimageset.h" #include "../imagesets/msoptions.h" #include "../imagesets/multibandmsimageset.h" #include "../util/logger.h" #include #include #include #include using imagesets::H5ImageSet; using imagesets::ImageSet; using imagesets::JoinedSPWSet; using imagesets::MSImageSet; // The order of initialization: // 1. If Lua strategy given, load Lua strategy // 2. Determine options // 3. For each run and each file: // a) If no Lua strategy was given, open file and load corresponding Lua // strategy b) Run strategy void Runner::Run() { std::map optionsForAllRuns; { LuaThreadGroup lua(1); loadStrategy(lua, _cmdLineOptions, std::unique_ptr()); Logger::Debug << "Loading options from strategy...\n"; optionsForAllRuns = OptionsFunction::GetOptions(lua.GetThread(0).State(), _cmdLineOptions); } // Let threadgroup go out of scope if (optionsForAllRuns.empty()) optionsForAllRuns.emplace("main", _cmdLineOptions); for (const std::pair& singleRunOptions : optionsForAllRuns) { Logger::Debug << "Starting run '" + singleRunOptions.first + "'...\n"; run(singleRunOptions.second); } } void Runner::loadStrategy( LuaThreadGroup& lua, const Options& options, const std::unique_ptr& imageSet) { if (!options.preamble.empty()) { Logger::Debug << "Running preample...\n"; lua.RunPreamble(options.preamble); } std::string executeFilename; // Execute filename override strategy filename, so that the options() function // can be in a different file than the execute functions. if (!options.executeFilename.empty()) executeFilename = options.executeFilename; if (!options.strategyFilename.empty()) executeFilename = options.strategyFilename; if (executeFilename.empty() && imageSet) { const std::string telescopeName = imageSet->TelescopeName(); const TelescopeFile::TelescopeId telescopeId = TelescopeFile::TelescopeIdFromName(telescopeName); if (telescopeId == TelescopeFile::GENERIC_TELESCOPE) { Logger::Warn << "**\n" "** Measurement set specified the following telescope name: '" << telescopeName << "'\n" "** No good strategy is known for this telescope!\n" "** A generic strategy will be used which might not be optimal.\n" "**\n"; } else { Logger::Info << "\nUsing a stock strategy that was optimized for the " "following telescope:\n" << "- " << TelescopeFile::TelescopeDescription(telescopeId) << "\n" << "Stock strategies might not perform well. It is " "recommended to make your own strategy\n" << "using the rfigui and specify it with the '-strategy " "...' parameter.\n\n"; } executeFilename = TelescopeFile::FindStrategy(telescopeId); if (executeFilename.empty()) { throw std::runtime_error("Could not find a strategy for telescope " + TelescopeFile::TelescopeName(telescopeId) + ".\n" "This strategy should have been installed when " "running 'make install'. Make sure\n" "aoflagger is properly installed."); } } if (!executeFilename.empty()) { try { Logger::Debug << "Opening strategy file '" << executeFilename << "'\n"; lua.LoadFile(executeFilename.c_str()); Logger::Debug << "Strategy parsed succesfully.\n"; } catch (std::exception& e) { throw std::runtime_error("ERROR: Reading strategy file \"" + executeFilename + "\" failed!\n" "\nThe thrown exception was:\n" + e.what() + "\n"); } } } static std::vector FilterProcessedFiles( const std::vector& ms_names) { std::vector result; std::copy_if(ms_names.begin(), ms_names.end(), std::back_inserter(result), [](const std::string& ms_name) { MSMetaData ms(ms_name); if (!ms.HasAOFlaggerHistory()) { return true; } Logger::Info << "Skipping " << ms_name << ",\n" "because the set contains AOFlagger history and " "-skip-flagged was given.\n"; return false; }); return result; } void Runner::run(const Options& options) { Logger::SetVerbosity(options.logVerbosity.value_or(Logger::NormalVerbosity)); const size_t threadCount = options.CalculateThreadCount(); Logger::Debug << "Number of threads: " << options.threadCount << "\n"; const std::vector& ms_files = options.skipFlagged ? FilterProcessedFiles(_cmdLineOptions.filenames) : _cmdLineOptions.filenames; if (_cmdLineOptions.concatenateFrequency && ms_files.size() > 1) { // Only use the multi-band image set when there at least 2 files. // Else just use the simpler code. processFrequencyConcatenatedFiles(options, ms_files, threadCount); } else { for (const std::string& filename : ms_files) { processFile(options, filename, threadCount); } } } std::unique_ptr Runner::initializeImageSet(const Options& options, FileOptions& fileOptions) { MSOptions msOptions; msOptions.ioMode = options.readMode.value_or(BaselineIOMode::AutoReadMode); msOptions.baselineIntegration = options.baselineIntegration; std::unique_ptr imageSet(ImageSet::Create( std::vector{fileOptions.filename}, msOptions)); if (H5ImageSet* h5ImageSet = dynamic_cast(imageSet.get()); h5ImageSet) { h5ImageSet->SetInterval(fileOptions.intervalStart, fileOptions.intervalEnd); } else if (MSImageSet* msImageSet = dynamic_cast(imageSet.get()); msImageSet) { if (options.dataColumn.empty()) msImageSet->SetDataColumnName("DATA"); else msImageSet->SetDataColumnName(options.dataColumn); msImageSet->SetReadUVW(options.readUVW.value_or(false)); // during the first iteration, the nr of intervals hasn't been calculated // yet. Do that now. if (fileOptions.intervalIndex == 0) { if (options.chunkSize != 0) { msImageSet->SetInterval(fileOptions.intervalStart, fileOptions.intervalEnd); const size_t obsTimesSize = msImageSet->MetaData().GetObservationTimes().size(); fileOptions.nIntervals = (obsTimesSize + options.chunkSize - 1) / options.chunkSize; Logger::Info << "Maximum interval size of " << options.chunkSize << " timesteps for total of " << obsTimesSize << " timesteps results in " << fileOptions.nIntervals << " intervals.\n"; if (options.startTimestep) fileOptions.resolvedIntStart = *options.startTimestep; else fileOptions.resolvedIntStart = 0; if (options.endTimestep) fileOptions.resolvedIntEnd = *options.endTimestep; else fileOptions.resolvedIntEnd = obsTimesSize + fileOptions.resolvedIntStart; } else { fileOptions.nIntervals = 1; } } if (fileOptions.nIntervals == 1) { msImageSet->SetInterval(fileOptions.intervalStart, fileOptions.intervalEnd); } else { const size_t nTimes = fileOptions.resolvedIntEnd - fileOptions.resolvedIntStart; size_t start = fileOptions.resolvedIntStart + fileOptions.intervalIndex * nTimes / fileOptions.nIntervals; size_t end = fileOptions.resolvedIntStart + (fileOptions.intervalIndex + 1) * nTimes / fileOptions.nIntervals; Logger::Info << "Starting flagging of interval " << fileOptions.intervalIndex << ", timesteps " << start << " - " << end << '\n'; msImageSet->SetInterval(start, end); } if (options.combineSPWs) { msImageSet->Initialize(); imageSet.release(); std::unique_ptr msImageSetPtr(msImageSet); imageSet.reset(new JoinedSPWSet(std::move(msImageSetPtr))); } } imageSet->Initialize(); return imageSet; } void Runner::processFile(const Options& options, const std::string& filename, size_t threadCount) { Logger::Info << "Starting strategy on " << to_simple_string( boost::posix_time::microsec_clock::local_time()) << '\n'; ScriptData scriptData; FileOptions fileOptions; fileOptions.intervalStart = options.startTimestep; fileOptions.intervalEnd = options.endTimestep; fileOptions.filename = filename; bool isMS = false; while (fileOptions.intervalIndex < fileOptions.nIntervals) { std::unique_ptr imageSet = initializeImageSet(options, fileOptions); isMS = dynamic_cast(imageSet.get()) != nullptr; LuaThreadGroup lua(threadCount); loadStrategy(lua, options, imageSet); std::mutex ioMutex; BaselineIterator blIterator(&ioMutex, options); blIterator.Run(*imageSet, lua, scriptData); ++fileOptions.intervalIndex; } if (isMS) writeHistory(options, filename); finishStatistics(filename, scriptData, isMS); } struct ChunkInfo { size_t start_time_step; size_t end_time_step; size_t n_chunks; size_t chunk_size; }; static std::optional GetChunkInfo(const Options& options, const std::string& ms_name) { if (!options.chunkSize) return {}; assert(options.startTimestep.has_value() == options.endTimestep.has_value() && "These fields should either both be set or both be unset."); ChunkInfo result; if (options.startTimestep) { Logger::Info << "Interval " << *options.startTimestep << ", " << *options.endTimestep << "\n"; result.start_time_step = *options.startTimestep; result.end_time_step = *options.endTimestep; } else { MSMetaData MetaData{ms_name}; result.start_time_step = 0; result.end_time_step = MetaData.TimestepCount(); } const size_t time_step_count = result.end_time_step - result.start_time_step; result.n_chunks = (time_step_count + options.chunkSize - 1) / options.chunkSize; result.chunk_size = (time_step_count + result.n_chunks - 1) / result.n_chunks; Logger::Info << "Chunking settings result in " << time_step_count << " intervals with " << result.chunk_size << " timesteps.\n"; if (result.n_chunks == 1) return {}; return result; } void Runner::processFrequencyConcatenatedFiles( Options options, const std::vector& ms_names, size_t n_threads) { Logger::Info << "Starting strategy on " << to_simple_string( boost::posix_time::microsec_clock::local_time()) << '\n'; // Unlike processing threads increasing the number of IO threads isn't always // beneficial. At some point adding additional IO threads decreases the // performance. The optimal number of IO threads is system dependent. Testing // on different systems showed the optimum to be between 16 and 32 threads. // These tests were executed in February 2022. constexpr size_t kMaxIoThreads = 16; if (n_threads == 0) n_threads = aocommon::system::ProcessorCount(); const size_t n_io_threads = std::min({kMaxIoThreads, n_threads, ms_names.size()}); int remaining_chunks = 1; const std::optional chunk_info = GetChunkInfo(options, ms_names.front()); // When the -chunk-size argument is used and more than one chunk is used the // interval is adjusted. if (chunk_info) { remaining_chunks = chunk_info->n_chunks; options.startTimestep = chunk_info->start_time_step; options.endTimestep = *options.startTimestep + chunk_info->chunk_size; } // The number of chunks to process is greater than or equal to one. // After processing the next iteration is prepared. Therefore the termination // condition is in the middle of the loop. while (true) { if (chunk_info) Logger::Info << "Starting flagging of interval " << 1 + (chunk_info->n_chunks - remaining_chunks) << ", timesteps " << *options.startTimestep << " - " << *options.endTimestep << '\n'; ProcessFrequencyConcatenatedFilesChunk(options, ms_names, n_threads, n_io_threads, chunk_info); if (--remaining_chunks; remaining_chunks == 0) break; *options.startTimestep += chunk_info->chunk_size; options.endTimestep = remaining_chunks == 1 ? chunk_info->end_time_step : *options.startTimestep + chunk_info->chunk_size; } for (const std::string& ms_name : ms_names) { writeHistory(options, ms_name); } } void Runner::ProcessFrequencyConcatenatedFilesChunk( const Options& options, const std::vector& ms_names, size_t n_threads, size_t n_io_threads, const std::optional& chunk_info) { const std::string data_column = options.dataColumn.empty() ? "DATA" : options.dataColumn; std::unique_ptr image_set = std::make_unique( ms_names, options.readMode.value_or(BaselineIOMode::AutoReadMode), data_column, options.startTimestep, options.endTimestep, n_io_threads); LuaThreadGroup thread_pool(n_threads); loadStrategy(thread_pool, options, image_set); std::mutex io_mutex; BaselineIterator baseline_iterator(&io_mutex, options); ScriptData script_data; baseline_iterator.Run(*image_set, thread_pool, script_data); static_cast(image_set.get()) ->WriteToMs(n_io_threads); if (script_data.GetStatistics()) Logger::Warn << "Statistics can't be written in multi-MS processing mode.\n" "Please remove collecting statistics from your Lua strategy.\n"; } void Runner::writeHistory(const Options& options, const std::string& filename) { MSMetaData ms(filename); Logger::Debug << "Adding strategy to history table of MS...\n"; try { std::string strategyFilename; if (!options.strategyFilename.empty()) { strategyFilename = options.strategyFilename; } // std::ifstream strategyFile(strategyFilename); // std::string content((std::istreambuf_iterator(strategyFile)), // (std::istreambuf_iterator()) ); ms.AddAOFlaggerHistory(strategyFilename, options.commandLine); } catch (std::exception& e) { Logger::Warn << "Failed to write history to MS: " << e.what() << '\n'; } } void Runner::finishStatistics(const std::string& filename, ScriptData& scriptData, bool isMS) { std::unique_ptr& statistics = scriptData.GetStatistics(); if (statistics) { if (isMS) { Logger::Debug << "Writing quality statistics to MS.\n"; QualityTablesFormatter qFormatter(filename); statistics->Save(qFormatter); } else { Logger::Warn << "Statistics can't be written when the input isn't a MS.\n" "Please remove collecting statistics from your Lua strategy.\n"; } } } aoflagger-v3.5.1/aoluarunner/runner.h0000664000175000017500000000300114752462134015776 0ustar oleole#ifndef RUNNER_H #define RUNNER_H #include "options.h" #include "../imagesets/imageset.h" #include #include #include #include struct ChunkInfo; class Runner { public: explicit Runner(const Options& cmdLineOptions) : _cmdLineOptions(cmdLineOptions) {} void Run(); private: struct FileOptions { std::string filename; size_t nIntervals = 1; size_t intervalIndex = 0; size_t resolvedIntStart = 0, resolvedIntEnd = 0; std::optional intervalStart, intervalEnd; }; void run(const Options& options); void loadStrategy(class LuaThreadGroup& lua, const Options& options, const std::unique_ptr& imageSet); void processFile(const Options& options, const std::string& filename, size_t threadCount); void processFrequencyConcatenatedFiles( Options options, const std::vector& filenames, size_t n_threads); void ProcessFrequencyConcatenatedFilesChunk( const Options& options, const std::vector& ms_names, size_t n_threads, size_t n_io_threads, const std::optional& chunk_info); std::unique_ptr initializeImageSet( const Options& options, FileOptions& fileOptions); void writeHistory(const Options& options, const std::string& filename); void finishStatistics(const std::string& filename, class ScriptData& scriptData, bool isMS); Options _cmdLineOptions; }; #endif aoflagger-v3.5.1/aoluarunner/baselineiterator.cpp0000664000175000017500000002577214752462134020377 0ustar oleole#include "baselineiterator.h" #include "../lua/luathreadgroup.h" #include "../lua/scriptdata.h" #include "../structures/antennainfo.h" #include "../util/logger.h" #include "../util/progress/dummyprogresslistener.h" #include "../util/stopwatch.h" #include "../imagesets/bhfitsimageset.h" #include "../imagesets/fitsimageset.h" #include "../imagesets/imageset.h" #include "../imagesets/msimageset.h" #include "../imagesets/filterbankset.h" #include "../imagesets/qualitystatimageset.h" #include "../imagesets/rfibaselineset.h" #include "writethread.h" #include #include #include BaselineIterator::BaselineIterator(std::mutex* ioMutex, const Options& options) : _options(options), _sequenceCount(0), _nextIndex(0), _threadCount(4), _loopIndex(), _ioMutex(ioMutex), _finishedBaselines(false), _exceptionOccured(false), _baselineProgress(0) {} BaselineIterator::~BaselineIterator() {} void BaselineIterator::Run(imagesets::ImageSet& imageSet, LuaThreadGroup& lua, ScriptData& scriptData) { _lua = &lua; _imageSet = &imageSet; _threadCount = _options.CalculateThreadCount(); _writeThread.reset(new WriteThread(imageSet, _threadCount, _ioMutex)); _globalScriptData = &scriptData; imagesets::MSImageSet* msImageSet = dynamic_cast(&imageSet); if (msImageSet) { // Check memory usage const imagesets::ImageSetIndex tempIndex = msImageSet->StartIndex(); const size_t timeStepCount = msImageSet->ObservationTimesVector(tempIndex).size(); const size_t channelCount = msImageSet->GetBandInfo(0).channels.size(); const double estMemorySizePerThread = 8.0 /*bp complex*/ * 4.0 /*polarizations*/ * double(timeStepCount) * double(channelCount) * 3.0 /* approx copies of the data that will be made in memory*/; Logger::Debug << "Estimate of memory each thread will use: " << memToStr(estMemorySizePerThread) << ".\n"; size_t compThreadCount = _threadCount; if (compThreadCount > 0) --compThreadCount; const int64_t memSize = aocommon::system::TotalMemory(); Logger::Debug << "Detected " << memToStr(memSize) << " of system memory.\n"; if (estMemorySizePerThread * double(compThreadCount) > memSize) { size_t maxThreads = size_t(memSize / estMemorySizePerThread); if (maxThreads < 1) maxThreads = 1; Logger::Warn << "This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" << _threadCount << " threads would require " << memToStr(estMemorySizePerThread * compThreadCount) << " of memory approximately.\n" "Number of threads that will actually be used: " << maxThreads << "\n" "This might hurt performance a lot!\n\n"; _threadCount = maxThreads; } } if (dynamic_cast(&imageSet) != nullptr && _threadCount != 1) { Logger::Info << "This is a Filterbank set -- disabling multi-threading\n"; _threadCount = 1; } if (!_options.antennaeToSkip.empty()) { Logger::Debug << "The following antennas will be skipped: "; for (const size_t a : _options.antennaeToSkip) Logger::Debug << a << ' '; Logger::Debug << '\n'; } if (!_options.antennaeToInclude.empty()) { Logger::Debug << "Only the following antennas will be included: "; for (const size_t a : _options.antennaeToInclude) Logger::Debug << a << ' '; Logger::Debug << '\n'; } _finishedBaselines = false; _sequenceCount = 0; _baselineProgress = 0; _nextIndex = 0; // Count the sequences that are to be processed imagesets::ImageSetIndex iteratorIndex = imageSet.StartIndex(); while (!iteratorIndex.HasWrapped()) { if (IsSequenceSelected(iteratorIndex)) ++_sequenceCount; iteratorIndex.Next(); } Logger::Debug << "Will process " << _sequenceCount << " sequences.\n"; // Initialize thread data and threads _loopIndex = imageSet.StartIndex(); std::vector threadGroup; const ReaderThread reader(*this); threadGroup.emplace_back(reader); for (unsigned i = 0; i < _threadCount; ++i) { const ProcessingThread function(*this, i); threadGroup.emplace_back(function); } for (std::thread& t : threadGroup) t.join(); _writeThread.reset(); if (_exceptionOccured) throw std::runtime_error( "An exception occured in the parallel (multi-threaded) processing of " "the baselines: the RFI strategy will not continue."); } bool BaselineIterator::IsSequenceSelected(imagesets::ImageSetIndex& index) { imagesets::IndexableSet* idImageSet = dynamic_cast(_imageSet); size_t a1id, a2id; if (idImageSet != nullptr) { a1id = idImageSet->GetAntenna1(index); a2id = idImageSet->GetAntenna2(index); if (!_options.bands.empty() && _options.bands.count(idImageSet->GetBand(index)) == 0) return false; if (!_options.fields.empty() && _options.fields.count(idImageSet->GetField(index)) == 0) return false; } else { a1id = 0; a2id = 0; } if (_options.antennaeToSkip.count(a1id) != 0 || _options.antennaeToSkip.count(a2id) != 0) return false; if (!_options.antennaeToInclude.empty() && (_options.antennaeToInclude.count(a1id) == 0 && _options.antennaeToInclude.count(a2id) == 0)) return false; // For SD/BHFits/QS/rfibl files, we want to select everything -- it's // confusing if the default option "only flag cross correlations" would also // hold for single-"baseline" files. if (!_imageSet->HasCrossCorrelations()) return true; switch (_options.baselineSelection.value_or( BaselineSelection::CrossCorrelations)) { case BaselineSelection::All: return true; case BaselineSelection::CrossCorrelations: { return a1id != a2id; } case BaselineSelection::AutoCorrelations: return a1id == a2id; case BaselineSelection::Current: case BaselineSelection::EqualToCurrent: case BaselineSelection::AutoCorrelationsOfCurrentAntennae: throw std::runtime_error("Not implemented"); } return false; } imagesets::ImageSetIndex BaselineIterator::GetNextIndex() { const std::lock_guard lock(_mutex); while (!_loopIndex.HasWrapped()) { if (IsSequenceSelected(_loopIndex)) { imagesets::ImageSetIndex newIndex(_loopIndex); _loopIndex.Next(); return newIndex; } _loopIndex.Next(); } return imagesets::ImageSetIndex(); } void BaselineIterator::SetExceptionOccured() { const std::lock_guard lock(_mutex); _exceptionOccured = true; _dataProcessed.notify_all(); _dataAvailable.notify_all(); } void BaselineIterator::SetFinishedBaselines() { const std::lock_guard lock(_mutex); _finishedBaselines = true; } void BaselineIterator::ProcessingThread::operator()() { ScriptData scriptData; const std::string executeFunctionName = _parent._options.executeFunctionName.empty() ? "execute" : _parent._options.executeFunctionName; try { std::unique_ptr baseline = _parent.GetNextBaseline(); while (baseline != nullptr) { /* TODO std::ostringstream progressStr; if(baseline->MetaData()->HasAntenna1() && baseline->MetaData()->HasAntenna2()) progressStr << "Processing baseline " << baseline->MetaData()->Antenna1().name << " x " << baseline->MetaData()->Antenna2().name; else progressStr << "Processing next baseline"; _parent.SetProgress(_progress, _parent.BaselineProgress(), _parent._baselineCount, progressStr.str(), _threadIndex); */ TimeFrequencyData data(baseline->Data()); _parent._lua->Execute(_threadIndex, data, baseline->MetaData(), scriptData, executeFunctionName); _parent._writeThread->SaveFlags(data, baseline->Index()); baseline = _parent.GetNextBaseline(); _parent.IncBaselineProgress(); } } catch (std::exception& e) { Logger::Error << e.what() << '\n'; _parent.SetExceptionOccured(); } { const std::unique_lock ioLock(*_parent._ioMutex); _parent._globalScriptData->Combine(std::move(scriptData)); } Logger::Debug << "Processing thread finished.\n"; } void BaselineIterator::ReaderThread::operator()() { Stopwatch watch(true); bool finished = false; const size_t threadCount = _parent._threadCount; size_t minRecommendedBufferSize, maxRecommendedBufferSize; imagesets::MSImageSet* msImageSet = dynamic_cast(_parent._imageSet); if (msImageSet != nullptr) { minRecommendedBufferSize = msImageSet->Reader()->GetMinRecommendedBufferSize(threadCount); maxRecommendedBufferSize = msImageSet->Reader()->GetMaxRecommendedBufferSize(threadCount) - _parent.GetBaselinesInBufferCount(); } else { minRecommendedBufferSize = 1; maxRecommendedBufferSize = 2; } do { watch.Pause(); _parent.WaitForReadBufferAvailable(minRecommendedBufferSize); if (!_parent._exceptionOccured) { const size_t wantedCount = maxRecommendedBufferSize - _parent.GetBaselinesInBufferCount(); size_t requestedCount = 0; std::unique_lock lock(*_parent._ioMutex); watch.Start(); for (size_t i = 0; i < wantedCount; ++i) { const imagesets::ImageSetIndex index = _parent.GetNextIndex(); if (!index.Empty()) { _parent._imageSet->AddReadRequest(index); ++requestedCount; } else { finished = true; break; } } if (requestedCount > 0) { DummyProgressListener dummy; _parent._imageSet->PerformReadRequests(dummy); watch.Pause(); for (size_t i = 0; i < requestedCount; ++i) { std::unique_ptr baseline = _parent._imageSet->GetNextRequested(); const std::lock_guard bufferLock(_parent._mutex); _parent._baselineBuffer.emplace(std::move(baseline)); } } lock.unlock(); _parent._dataAvailable.notify_all(); } watch.Start(); } while (!finished && !_parent._exceptionOccured); _parent.SetFinishedBaselines(); _parent._dataAvailable.notify_all(); watch.Pause(); Logger::Debug << "Time spent on reading: " << watch.ToString() << '\n'; } std::string BaselineIterator::memToStr(double memSize) { std::ostringstream str; if (memSize > 1024.0 * 1024.0 * 1024.0 * 1024.0) str << round(memSize * 10.0 / (1024.0 * 1024.0 * 1024.0 * 1024.0)) / 10.0 << " TB"; else if (memSize > 1024.0 * 1024.0 * 1024.0) str << round(memSize * 10.0 / (1024.0 * 1024.0 * 1024.0)) / 10.0 << " GB"; else if (memSize > 1024.0 * 1024.0) str << round(memSize * 10.0 / (1024.0 * 1024.0)) / 10.0 << " MB"; else if (memSize > 1024.0) str << round(memSize * 10.0 / (1024.0)) / 10.0 << " KB"; else str << memSize << " B"; return str.str(); } aoflagger-v3.5.1/aoluarunner/options.h0000664000175000017500000001426715065216451016175 0ustar oleole#ifndef OPTIONS_H #define OPTIONS_H #include "../structures/types.h" #include "../util/logger.h" #include #include #include #include #include enum class BaselineSelection { All, CrossCorrelations, AutoCorrelations, EqualToCurrent, AutoCorrelationsOfCurrentAntennae, Current }; enum class BaselineIntegrationMode { Count, Average, AverageAbs, Squared, Stddev }; /// Convert a baseline integration mode to a descriptive string inline std::string ToString(BaselineIntegrationMode mode) { switch (mode) { case BaselineIntegrationMode::Count: return "count"; case BaselineIntegrationMode::Average: return "average"; case BaselineIntegrationMode::AverageAbs: return "absolute average"; case BaselineIntegrationMode::Squared: return "squared"; case BaselineIntegrationMode::Stddev: return "standard deviation"; } assert(false); return ""; } enum class BaselineIntegrationDifferencing { NoDifference, TimeDifference, FrequencyDifference }; /// Convert a differencing mode to a descriptive string inline std::string ToString(BaselineIntegrationDifferencing differencing) { switch (differencing) { case BaselineIntegrationDifferencing::NoDifference: return "no differencing"; case BaselineIntegrationDifferencing::TimeDifference: return "time difference"; case BaselineIntegrationDifferencing::FrequencyDifference: return "channel difference"; } assert(false); return ""; } struct BaselineIntegration { std::optional enable, withAutos, withFlagged; std::optional mode; std::optional differencing; /** * All options that are set in @ref other will override the settings * in this. */ void Override(const BaselineIntegration& other) { if (other.enable) enable = other.enable; if (other.withAutos) withAutos = other.withAutos; if (other.withFlagged) withFlagged = other.withFlagged; if (other.mode) mode = other.mode; if (other.differencing) differencing = other.differencing; } bool operator==(const BaselineIntegration& other) const { return enable == other.enable && withAutos == other.withAutos && withFlagged == other.withFlagged && mode == other.mode && differencing == other.differencing; } }; struct Options { std::set antennaeToInclude, antennaeToSkip; std::set bands; std::optional baselineSelection; BaselineIntegration baselineIntegration; size_t chunkSize; std::optional combineSPWs; std::optional concatenateFrequency; std::string dataColumn; std::string executeFilename; std::string executeFunctionName; std::set fields; std::optional readMode; std::optional readUVW; std::string scriptVersion; std::optional skipFlagged; std::optional startTimestep, endTimestep; std::string strategyFilename; size_t threadCount; std::optional logVerbosity; std::vector preamble; std::string commandLine; std::vector filenames; Options() : chunkSize(0), threadCount(0) {} /** * All options that are set in @ref other will override the settings * in this. */ void Override(const Options& other) { if (!other.antennaeToInclude.empty()) antennaeToInclude = other.antennaeToInclude; if (!other.antennaeToSkip.empty()) antennaeToSkip = other.antennaeToSkip; if (!other.bands.empty()) bands = other.bands; baselineIntegration.Override(other.baselineIntegration); if (other.baselineSelection) baselineSelection = other.baselineSelection; if (other.chunkSize) chunkSize = other.chunkSize; if (other.combineSPWs) combineSPWs = other.combineSPWs; if (other.concatenateFrequency) concatenateFrequency = other.concatenateFrequency; if (!other.dataColumn.empty()) dataColumn = other.dataColumn; if (!other.executeFilename.empty()) executeFilename = other.executeFilename; if (!other.executeFunctionName.empty()) executeFunctionName = other.executeFunctionName; if (!other.fields.empty()) fields = other.fields; if (other.readMode) readMode = other.readMode; if (other.readUVW) readUVW = other.readUVW; if (!other.scriptVersion.empty()) scriptVersion = other.scriptVersion; if (other.skipFlagged) skipFlagged = other.skipFlagged; if (other.startTimestep) startTimestep = other.startTimestep; if (other.endTimestep) endTimestep = other.endTimestep; if (!other.strategyFilename.empty()) strategyFilename = other.strategyFilename; if (other.threadCount) threadCount = other.threadCount; if (other.logVerbosity) logVerbosity = other.logVerbosity; preamble.insert(preamble.begin(), other.preamble.begin(), other.preamble.end()); if (!other.commandLine.empty()) commandLine = other.commandLine; if (!other.filenames.empty()) filenames = other.filenames; } bool operator==(const Options& rhs) const { return antennaeToInclude == rhs.antennaeToInclude && antennaeToSkip == rhs.antennaeToSkip && bands == rhs.bands && baselineIntegration == rhs.baselineIntegration && baselineSelection == rhs.baselineSelection && chunkSize == rhs.chunkSize && combineSPWs == rhs.combineSPWs && concatenateFrequency == rhs.concatenateFrequency && dataColumn == rhs.dataColumn && executeFilename == rhs.executeFilename && executeFunctionName == rhs.executeFunctionName && fields == rhs.fields && readMode == rhs.readMode && readUVW == rhs.readUVW && scriptVersion == rhs.scriptVersion && skipFlagged == rhs.skipFlagged && startTimestep == rhs.startTimestep && endTimestep == rhs.endTimestep && strategyFilename == rhs.strategyFilename && threadCount == rhs.threadCount && logVerbosity == rhs.logVerbosity && preamble == rhs.preamble && commandLine == rhs.commandLine && filenames == rhs.filenames; } bool operator!=(const Options& rhs) const { return !(*this == rhs); } size_t CalculateThreadCount() const; }; #endif aoflagger-v3.5.1/aoluarunner/writethread.cpp0000664000175000017500000000617314752462134017357 0ustar oleole#include "writethread.h" #include "../imagesets/multibandmsimageset.h" #include "../util/logger.h" WriteThread::WriteThread(imagesets::ImageSet& imageSet, size_t calcThreadCount, std::mutex* ioMutex) : _ioMutex(ioMutex), _isWriteFinishing(false), _maxWriteBufferItems(calcThreadCount * 5), _minWriteBufferItemsForWriting(calcThreadCount * 4) { FlushThread flushFunction; flushFunction._parent = this; if (dynamic_cast(&imageSet)) { // TODO Would this method also be safe for other writers? _flusher.reset(new std::thread(flushFunction, &imageSet)); } else { std::unique_lock iolock(*_ioMutex); std::unique_ptr localImageSet = imageSet.Clone(); iolock.unlock(); _flusher.reset(new std::thread(flushFunction, std::move(localImageSet))); } } WriteThread::~WriteThread() { std::unique_lock lock(_writeMutex); _isWriteFinishing = true; _writeBufferChange.notify_all(); lock.unlock(); Logger::Debug << "Finishing the flusher thread...\n"; _flusher->join(); } void WriteThread::SaveFlags(const TimeFrequencyData& data, imagesets::ImageSetIndex& imageSetIndex) { std::vector masks; if (data.MaskCount() <= 1) masks.emplace_back(data.GetSingleMask()); else for (size_t i = 0; i < data.MaskCount(); ++i) { masks.emplace_back(data.GetMask(i)); } const BufferItem newItem(masks, imageSetIndex); pushInWriteBuffer(newItem); } void WriteThread::pushInWriteBuffer(const BufferItem& newItem) { std::unique_lock lock(_writeMutex); while (_writeBuffer.size() >= _maxWriteBufferItems) _writeBufferChange.wait(lock); _writeBuffer.emplace(newItem); _writeBufferChange.notify_all(); } void WriteThread::FlushThread::operator()( std::unique_ptr imageSet) { operator()(imageSet.get()); } void WriteThread::FlushThread::operator()(imagesets::ImageSet* imageSet) { std::unique_lock lock(_parent->_writeMutex); do { while (_parent->_writeBuffer.size() < _parent->_minWriteBufferItemsForWriting && !_parent->_isWriteFinishing) _parent->_writeBufferChange.wait(lock); std::stack bufferCopy; while (!_parent->_writeBuffer.empty()) { const BufferItem item = _parent->_writeBuffer.top(); _parent->_writeBuffer.pop(); bufferCopy.push(item); } _parent->_writeBufferChange.notify_all(); if (bufferCopy.size() >= _parent->_minWriteBufferItemsForWriting) Logger::Debug << "Flag buffer has reached minimal writing size, flushing " "flags...\n"; else Logger::Debug << "Flushing flags...\n"; lock.unlock(); std::unique_lock ioLock(*_parent->_ioMutex); while (!bufferCopy.empty()) { BufferItem item = bufferCopy.top(); bufferCopy.pop(); imageSet->AddWriteFlagsTask(item._index, item._masks); } imageSet->PerformWriteFlagsTask(); ioLock.unlock(); lock.lock(); } while (!_parent->_isWriteFinishing || !_parent->_writeBuffer.empty()); } aoflagger-v3.5.1/aoluarunner/options.cpp0000664000175000017500000000030614752462134016520 0ustar oleole#include "options.h" #include size_t Options::CalculateThreadCount() const { if (threadCount) return threadCount; else return aocommon::system::ProcessorCount(); } aoflagger-v3.5.1/aoluarunner/writethread.h0000664000175000017500000000230514752462134017015 0ustar oleole#ifndef AOLUA_WRITE_THREAD_H #define AOLUA_WRITE_THREAD_H #include "../imagesets/imageset.h" #include #include #include #include #include class WriteThread { public: WriteThread(imagesets::ImageSet& imageSet, size_t calcThreadCount, std::mutex* ioMutex); ~WriteThread(); void SaveFlags(const TimeFrequencyData& data, imagesets::ImageSetIndex& imageSetIndex); private: struct FlushThread { WriteThread* _parent; void operator()(std::unique_ptr imageSet); void operator()(imagesets::ImageSet* imageSet); }; struct BufferItem { BufferItem(const std::vector& masks, const imagesets::ImageSetIndex& index) : _masks(masks), _index(index) {} std::vector _masks; imagesets::ImageSetIndex _index; }; void pushInWriteBuffer(const BufferItem& newItem); std::mutex _writeMutex, *_ioMutex; std::condition_variable _writeBufferChange; std::unique_ptr _flusher; bool _isWriteFinishing; size_t _maxWriteBufferItems; size_t _minWriteBufferItemsForWriting; std::stack _writeBuffer; }; #endif aoflagger-v3.5.1/cmake/0000775000175000017500000000000015146315735013051 5ustar oleoleaoflagger-v3.5.1/cmake/FindCasacore.cmake0000664000175000017500000002500615015560052016364 0ustar oleole# * Try to find Casacore include dirs and libraries Usage: find_package(Casacore # [REQUIRED] [COMPONENTS components...]) Valid components are: casa, # coordinates, derivedmscal, fits, images, lattices, meas, measures, mirlib, # ms, msfits, python, scimath, scimath_f, tables # # Note that most components are dependent on other (more basic) components. In # that case, it suffices to specify the "top-level" components; dependent # components will be searched for automatically. # # The dependency tree can be generated using the script get_casacore_deps.sh. # For this, you need to have a complete casacore installation, built with shared # libraries, at your disposal. # # The dependencies in this macro were generated against casacore release 1.7.0. # # Variables used by this module: CASACORE_ROOT_DIR - Casacore root # directory. # # Variables defined by this module: CASACORE_FOUND - System has # Casacore, which means that the include dir was found, as well as all libraries # specified (not cached) CASACORE_INCLUDE_DIR - Casacore include directory # (cached) CASACORE_INCLUDE_DIRS - Casacore include directories (not cached) # identical to CASACORE_INCLUDE_DIR CASACORE_LIBRARIES - The Casacore # libraries (not cached) CASA_${COMPONENT}_LIBRARY - The absolute path of # Casacore library "component" (cached) HAVE_AIPSPP - True if # system has Casacore (cached) for backward compatibility with AIPS++ # HAVE_CASACORE - True if system has Casacore (cached) identical to # CASACORE_FOUND TAQL_EXECUTABLE - The absolute path of the TaQL # executable (cached) # # ATTENTION: The component names need to be in lower case, just as the casacore # library names. However, the CMake variables use all upper case. # Copyright (C) 2009 ASTRON (Netherlands Institute for Radio Astronomy) P.O.Box # 2, 7990 AA Dwingeloo, The Netherlands # # This file is part of the LOFAR software suite. The LOFAR software suite is # free software: you can redistribute it and/or modify it under the terms of the # GNU General Public License as published by the Free Software Foundation, # either version 3 of the License, or (at your option) any later version. # # The LOFAR software suite is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along with # the LOFAR software suite. If not, see . # # $Id: FindCasacore.cmake 31487 2015-04-16 11:28:17Z dijkema $ # * casacore_resolve_dependencies(_result) # # Resolve the Casacore library dependencies for the given components. The list # of dependent libraries will be returned in the variable result. It is sorted # from least dependent to most dependent library, so it can be directly fed to # the linker. # # Usage: casacore_resolve_dependencies(result components...) # macro(casacore_resolve_dependencies _result) set(${_result} ${ARGN}) set(_index 0) # Do a breadth-first search through the dependency graph; append to the result # list the dependent components for each item in that list. Duplicates will be # removed later. while(1) list(LENGTH ${_result} _length) if(NOT _index LESS _length) break() endif(NOT _index LESS _length) list(GET ${_result} ${_index} item) list(APPEND ${_result} ${Casacore_${item}_DEPENDENCIES}) math(EXPR _index "${_index}+1") endwhile(1) # Remove all duplicates in the current result list, while retaining only the # last of each duplicate. list(REVERSE ${_result}) list(REMOVE_DUPLICATES ${_result}) list(REVERSE ${_result}) endmacro(casacore_resolve_dependencies _result) # * casacore_find_library(_name) # # Search for the library ${_name}. If library is found, add it to # CASACORE_LIBRARIES; if not, add ${_name} to CASACORE_MISSING_COMPONENTS and # set CASACORE_FOUND to false. # # Usage: casacore_find_library(name) # macro(casacore_find_library _name) string(TOUPPER ${_name} _NAME) find_library( ${_NAME}_LIBRARY ${_name} HINTS ${CASACORE_ROOT_DIR} PATH_SUFFIXES lib) mark_as_advanced(${_NAME}_LIBRARY) if(${_NAME}_LIBRARY) list(APPEND CASACORE_LIBRARIES ${${_NAME}_LIBRARY}) else(${_NAME}_LIBRARY) set(CASACORE_FOUND FALSE) list(APPEND CASACORE_MISSING_COMPONENTS ${_name}) endif(${_NAME}_LIBRARY) endmacro(casacore_find_library _name) # * casacore_find_package(_name) # # Search for the package ${_name}. If the package is found, add the contents of # ${_name}_INCLUDE_DIRS to CASACORE_INCLUDE_DIRS and ${_name}_LIBRARIES to # CASACORE_LIBRARIES. # # If Casacore itself is required, then, strictly speaking, the packages it # requires must be present. However, when linking against static libraries they # may not be needed. One can override the REQUIRED setting by switching # CASACORE_MAKE_REQUIRED_EXTERNALS_OPTIONAL to ON. Beware that this might cause # compile and/or link errors. # # Usage: casacore_find_package(name [REQUIRED]) # macro(casacore_find_package _name) if("${ARGN}" MATCHES "^REQUIRED$" AND Casacore_FIND_REQUIRED AND NOT CASACORE_MAKE_REQUIRED_EXTERNALS_OPTIONAL) find_package(${_name} REQUIRED) else() find_package(${_name}) endif() if(${_name}_FOUND) list(APPEND CASACORE_INCLUDE_DIRS ${${_name}_INCLUDE_DIRS}) list(APPEND CASACORE_LIBRARIES ${${_name}_LIBRARIES}) endif(${_name}_FOUND) endmacro(casacore_find_package _name) # Define the Casacore components. set(Casacore_components casa coordinates derivedmscal fits images lattices meas measures mirlib ms msfits python scimath scimath_f tables) # Define the Casacore components' inter-dependencies. set(Casacore_casa_DEPENDENCIES) set(Casacore_coordinates_DEPENDENCIES fits measures casa) set(Casacore_derivedmscal_DEPENDENCIES ms measures tables casa) set(Casacore_fits_DEPENDENCIES measures tables casa) set(Casacore_images_DEPENDENCIES mirlib lattices coordinates fits measures scimath tables casa) set(Casacore_lattices_DEPENDENCIES tables scimath casa) set(Casacore_meas_DEPENDENCIES measures tables casa) set(Casacore_measures_DEPENDENCIES tables casa) set(Casacore_mirlib_DEPENDENCIES) set(Casacore_ms_DEPENDENCIES measures scimath tables casa) set(Casacore_msfits_DEPENDENCIES ms fits measures tables casa) set(Casacore_python_DEPENDENCIES casa) set(Casacore_scimath_DEPENDENCIES scimath_f casa) set(Casacore_scimath_f_DEPENDENCIES) set(Casacore_tables_DEPENDENCIES casa) # Initialize variables. set(CASACORE_FOUND FALSE) set(CASACORE_DEFINITIONS) set(CASACORE_LIBRARIES) set(CASACORE_MISSING_COMPONENTS) # Search for the header file first. if(NOT CASACORE_INCLUDE_DIR) find_path( CASACORE_INCLUDE_DIR casacore/casa/aips.h HINTS ${CASACORE_ROOT_DIR} PATH_SUFFIXES include) mark_as_advanced(CASACORE_INCLUDE_DIR) endif(NOT CASACORE_INCLUDE_DIR) # Fallback for systems that have old casacore installed in directory not called # 'casacore' This fallback can be removed once we move to casacore 2.0 which # always puts headers in 'casacore' if(NOT CASACORE_INCLUDE_DIR) find_path( CASACORE_INCLUDE_DIR casa/aips.h HINTS ${CASACORE_ROOT_DIR} PATH_SUFFIXES include) mark_as_advanced(CASACORE_INCLUDE_DIR) endif(NOT CASACORE_INCLUDE_DIR) if(NOT CASACORE_INCLUDE_DIR) set(CASACORE_ERROR_MESSAGE "Casacore: unable to find the header file casa/aips.h.\nPlease set CASACORE_ROOT_DIR to the root directory containing Casacore." ) else(NOT CASACORE_INCLUDE_DIR) # We've found the header file; let's continue. set(CASACORE_FOUND TRUE) # Note that new Casacore uses #include, while LOFAR still # uses #include. Hence use both in -I path. set(CASACORE_INCLUDE_DIRS ${CASACORE_INCLUDE_DIR} ${CASACORE_INCLUDE_DIR}/casacore) # Search for some often used binaries. find_program(TAQL_EXECUTABLE taql HINTS ${CASACORE_ROOT_DIR}/bin) mark_as_advanced(TAQL_EXECUTABLE) # If the user specified components explicity, use that list; otherwise we'll # assume that the user wants to use all components. if(NOT Casacore_FIND_COMPONENTS) set(Casacore_FIND_COMPONENTS ${Casacore_components}) endif(NOT Casacore_FIND_COMPONENTS) # Get a list of all dependent Casacore libraries that need to be found. casacore_resolve_dependencies(_find_components ${Casacore_FIND_COMPONENTS}) # Find the library for each component, and handle external dependencies foreach(_comp ${_find_components}) casacore_find_library(casa_${_comp}) if(${_comp} STREQUAL casa) # Use 'find_package' directly for HDF5, since 'casacore_find_package' does # not support the extra 'COMPONENTS CXX' arguments. find_package(HDF5 COMPONENTS CXX) if(HDF5_FOUND) list(APPEND CASACORE_INCLUDE_DIRS ${HDF5_INCLUDE_DIRS}) list(APPEND CASACORE_LIBRARIES ${HDF5_LIBRARIES}) endif(HDF5_FOUND) casacore_find_library(m) list(APPEND CASACORE_LIBRARIES ${CMAKE_DL_LIBS}) elseif(${_comp} STREQUAL coordinates) casacore_find_package(WCSLIB REQUIRED) elseif(${_comp} STREQUAL fits) casacore_find_package(CFITSIO REQUIRED) elseif(${_comp} STREQUAL scimath_f) casacore_find_package(LAPACK REQUIRED) endif(${_comp} STREQUAL casa) endforeach(_comp ${_find_components}) endif(NOT CASACORE_INCLUDE_DIR) # Set HAVE_CASACORE; and HAVE_AIPSPP (for backward compatibility with AIPS++). if(CASACORE_FOUND) set(HAVE_CASACORE TRUE CACHE INTERNAL "Define if Casacore is installed") set(HAVE_AIPSPP TRUE CACHE INTERNAL "Define if AIPS++/Casacore is installed") endif(CASACORE_FOUND) # Compose diagnostic message if not all necessary components were found. if(CASACORE_MISSING_COMPONENTS) set(CASACORE_ERROR_MESSAGE "Casacore: the following components could not be found:\n ${CASACORE_MISSING_COMPONENTS}" ) endif(CASACORE_MISSING_COMPONENTS) # Print diagnostics. if(CASACORE_FOUND) if(NOT Casacore_FIND_QUIETLY) message(STATUS "Found the following Casacore components: ") foreach(_comp ${_find_components}) string(TOUPPER casa_${_comp} _COMP) message(STATUS " ${_comp}: ${${_COMP}_LIBRARY}") endforeach(_comp ${_find_components}) endif(NOT Casacore_FIND_QUIETLY) else(CASACORE_FOUND) if(Casacore_FIND_REQUIRED) message(FATAL_ERROR "${CASACORE_ERROR_MESSAGE}") else(Casacore_FIND_REQUIRED) message(STATUS "${CASACORE_ERROR_MESSAGE}") endif(Casacore_FIND_REQUIRED) endif(CASACORE_FOUND) aoflagger-v3.5.1/cmake/FindCFITSIO.cmake0000664000175000017500000000377214752462134015763 0ustar oleole# * Try to find CFITSIO. Variables used by this module: CFITSIO_ROOT_DIR - # CFITSIO root directory Variables defined by this module: CFITSIO_FOUND - # system has CFITSIO CFITSIO_INCLUDE_DIR - the CFITSIO include directory # (cached) CFITSIO_INCLUDE_DIRS - the CFITSIO include directories (identical # to CFITSIO_INCLUDE_DIR) CFITSIO_LIBRARY - the CFITSIO library (cached) # CFITSIO_LIBRARIES - the CFITSIO libraries (identical to CFITSIO_LIBRARY) # Copyright (C) 2009 ASTRON (Netherlands Institute for Radio Astronomy) P.O.Box # 2, 7990 AA Dwingeloo, The Netherlands # # This file is part of the LOFAR software suite. The LOFAR software suite is # free software: you can redistribute it and/or modify it under the terms of the # GNU General Public License as published by the Free Software Foundation, # either version 3 of the License, or (at your option) any later version. # # The LOFAR software suite is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along with # the LOFAR software suite. If not, see . # # $Id: FindCFITSIO.cmake 22498 2012-10-23 10:51:12Z loose $ if(NOT CFITSIO_FOUND) find_path( CFITSIO_INCLUDE_DIR fitsio.h HINTS ${CFITSIO_ROOT_DIR} PATH_SUFFIXES include include/cfitsio include/libcfitsio0) find_library( CFITSIO_LIBRARY cfitsio HINTS ${CFITSIO_ROOT_DIR} PATH_SUFFIXES lib) find_library(M_LIBRARY m) mark_as_advanced(CFITSIO_INCLUDE_DIR CFITSIO_LIBRARY M_LIBRARY) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(CFITSIO DEFAULT_MSG CFITSIO_LIBRARY M_LIBRARY CFITSIO_INCLUDE_DIR) set(CFITSIO_INCLUDE_DIRS ${CFITSIO_INCLUDE_DIR}) set(CFITSIO_LIBRARIES ${CFITSIO_LIBRARY} ${M_LIBRARY}) endif(NOT CFITSIO_FOUND) aoflagger-v3.5.1/cmake/config/0000775000175000017500000000000015146315735014316 5ustar oleoleaoflagger-v3.5.1/cmake/config/aoflagger-config-version.cmake.in0000664000175000017500000000141314752462134022577 0ustar oleoleset(PACKAGE_VERSION "@AOFLAGGER_VERSION@") # Versions are compatible if version found >= necessary AND the major # version number matches. Hence, requiring aoflagger 2.10 would not # be compatible with a found version of 3.0. if("@AOFLAGGER_VERSION@" VERSION_LESS PACKAGE_FIND_VERSION OR NOT "@AOFLAGGER_VERSION_MAJOR@" VERSION_EQUAL PACKAGE_FIND_VERSION_MAJOR) message(STATUS "Found an incompatible version of AOFlagger: found @AOFLAGGER_VERSION@, need ${PACKAGE_FIND_VERSION}") set(PACKAGE_VERSION_COMPATIBLE FALSE) set(PACKAGE_VERSION_EXACT FALSE) else() set(PACKAGE_VERSION_COMPATIBLE TRUE) if("@AOFLAGGER_VERSION@" VERSION_EQUAL PACKAGE_FIND_VERSION) set(PACKAGE_VERSION_EXACT TRUE) else() set(PACKAGE_VERSION_EXACT FALSE) endif() endif() aoflagger-v3.5.1/cmake/config/aoflagger-config.cmake.in0000664000175000017500000000226014752462134021115 0ustar oleole# This is the cmake config script for AOFlagger. # # It sets the following variables: # - AOFLAGGER_INCLUDE_DIR # - AOFLAGGER_LIB # - AOFLAGGER_LIB_PATH # - AOFLAGGER_VERSION[_MAJOR/_MINOR] # - AOFLAGGER_FOUND # - AOFLAGGER_ROOT_DIR # Compute path get_filename_component(_AOFLAGGER_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) get_filename_component(_AOFLAGGER_CMAKE_DIR_ABS "${_AOFLAGGER_CMAKE_DIR}" ABSOLUTE) get_filename_component(_AOFLAGGER_ROOT_DIR "${_AOFLAGGER_CMAKE_DIR_ABS}/../../.." ABSOLUTE) set(AOFLAGGER_ROOT_DIR "${_AOFLAGGER_ROOT_DIR}" CACHE PATH "AOFlagger root (prefix) directory") set(AOFLAGGER_INCLUDE_DIR "${AOFLAGGER_ROOT_DIR}/include" CACHE PATH "AOFlagger include directory") set(AOFLAGGER_LIB_PATH "${AOFLAGGER_ROOT_DIR}/lib" CACHE PATH "AOFlagger library directory") find_library(AOFLAGGER_LIB aoflagger PATH ${AOFLAGGER_LIB_PATH} NO_DEFAULT_PATH DOC "AOFlagger library directory") message(STATUS "Found AOFlagger @AOFLAGGER_VERSION@.") message(STATUS " AOFlagger include dir: ${AOFLAGGER_INCLUDE_DIR}") message(STATUS " AOFlagger lib: ${AOFLAGGER_LIB}") unset(_AOFLAGGER_ROOT_DIR) unset(_AOFLAGGER_CMAKE_DIR) unset(_AOFLAGGER_CMAKE_DIR_ABS) aoflagger-v3.5.1/version.h.in0000664000175000017500000000066714752462134014243 0ustar oleole#ifndef AOFLAGGER_VERSION_H #define AOFLAGGER_VERSION_H #define AOFLAGGER_VERSION_STR "@AOFLAGGER_VERSION_STR@" #define AOFLAGGER_VERSION_MAJOR @AOFLAGGER_VERSION_MAJOR@ #define AOFLAGGER_VERSION_MINOR @AOFLAGGER_VERSION_MINOR@ #define AOFLAGGER_VERSION_SUBMINOR @AOFLAGGER_VERSION_SUBMINOR@ #define AOFLAGGER_VERSION_DATE_STR "@AOFLAGGER_VERSION_DATE_STR@" #define AOFLAGGER_INSTALL_PATH "@CMAKE_INSTALL_PREFIX@" #endif aoflagger-v3.5.1/.readthedocs.yaml0000664000175000017500000000050014752462134015211 0ustar oleole# Read the Docs configuration file for Sphinx projects # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details version: 2 build: os: ubuntu-22.04 tools: python: "3.11" sphinx: configuration: doc/source/conf.py python: install: - requirements: doc/source/piprequirements.txt aoflagger-v3.5.1/lua/0000775000175000017500000000000015146315735012552 5ustar oleoleaoflagger-v3.5.1/lua/luastrategy.cpp0000664000175000017500000001506615131147277015630 0ustar oleole#include "luastrategy.h" #include #include "datawrapper.h" #include "functions.h" #include "functionswrapper.h" #include "scriptdata.h" #include "tools.h" LuaStrategy::LuaStrategy() : _state(luaL_newstate()) {} LuaStrategy::LuaStrategy(LuaStrategy&& source) : _state(source._state) { source._state = nullptr; } LuaStrategy& LuaStrategy::operator=(LuaStrategy&& source) { std::swap(_state, source._state); return *this; } void LuaStrategy::clear() { if (_state) lua_close(_state); _state = nullptr; } void LuaStrategy::check(lua_State* state, int error) { if (error) { const char* msg = lua_tostring(state, -1); if (msg) { const std::string str(msg); lua_pop(state, 1); // pop error throw std::runtime_error(str); } else { throw std::runtime_error("An unspecified error was thrown inside Lua"); } } } void LuaStrategy::loadaoflagger() { luaL_newmetatable(_state, "AOFlaggerData"); // Allow object access for AOFlaggerData structures lua_pushstring(_state, "__index"); lua_pushvalue(_state, -2); /* pushes the AOFlaggerData metatable */ lua_settable(_state, -3); /* metatable.__index = metatable */ static const struct luaL_Reg aofdatamembers[] = { {"clear_mask", Data::clear_mask}, {"convert_to_complex", Data::convert_to_complex}, {"convert_to_polarization", Data::convert_to_polarization}, {"copy", Data::copy}, {"flag_zeros", Data::flag_zeros}, {"flag_nans", Data::flag_nans}, {"get_antenna1_index", Data::get_antenna1_index}, {"get_antenna1_name", Data::get_antenna1_name}, {"get_antenna2_index", Data::get_antenna2_index}, {"get_antenna2_name", Data::get_antenna2_name}, {"get_antenna2_name", Data::get_antenna2_name}, {"get_baseline_angle", Data::get_baseline_angle}, {"get_baseline_distance", Data::get_baseline_distance}, {"get_baseline_vector", Data::get_baseline_vector}, {"get_complex_state", Data::get_complex_state}, {"get_frequencies", Data::get_frequencies}, {"get_polarizations", Data::get_polarizations}, {"get_times", Data::get_times}, {"has_metadata", Data::has_metadata}, {"invert_mask", Data::invert_mask}, {"is_auto_correlation", Data::is_auto_correlation}, {"is_complex", Data::is_complex}, {"join_mask", Data::join_mask}, {"set_mask", Data::set_mask}, {"set_mask_for_channel_range", Data::set_mask_for_channel_range}, {"set_masked_visibilities", Data::set_masked_visibilities}, {"set_polarization_data", Data::set_polarization_data}, {"set_visibilities", Data::set_visibilities}, {"__div", Data::div}, {"__gc", Data::gc}, {"__sub", Data::sub}, {nullptr, nullptr}}; luaL_setfuncs(_state, aofdatamembers, 0); // New table for package lua_newtable(_state); static const struct luaL_Reg aoflib[] = { {"apply_bandpass", Functions::apply_bandpass}, {"collect_statistics", Functions::collect_statistics}, {"copy_to_channel", Functions::copy_to_channel}, {"copy_to_frequency", Functions::copy_to_frequency}, {"downsample", Functions::downsample}, {"high_pass_filter", Functions::high_pass_filter}, {"low_pass_filter", Functions::low_pass_filter}, {"norm", Functions::norm}, {"normalize_bandpass", Functions::normalize_bandpass}, {"normalize_subbands", Functions::normalize_subbands}, {"print_polarization_statistics", Functions::print_polarization_statistics}, {"require_min_version", Functions::require_min_version}, {"require_max_version", Functions::require_max_version}, {"save_heat_map", Functions::save_heat_map}, {"scale_invariant_rank_operator", Functions::scale_invariant_rank_operator}, {"scale_invariant_rank_operator_masked", Functions::scale_invariant_rank_operator_masked}, {"set_progress", Functions::set_progress}, {"set_progress_text", Functions::set_progress_text}, {"sqrt", Functions::sqrt}, {"sumthreshold", Functions::sumthreshold}, {"sumthreshold_masked", Functions::sumthreshold_masked}, {"threshold_channel_rms", Functions::threshold_channel_rms}, {"threshold_timestep_rms", Functions::threshold_timestep_rms}, {"trim_channels", Functions::trim_channels}, {"trim_frequencies", Functions::trim_frequencies}, {"upsample", Functions::upsample_image}, // DEPRECATED {"upsample_image", Functions::upsample_image}, {"upsample_mask", Functions::upsample_mask}, {"visualize", Functions::visualize}, {nullptr, nullptr}}; luaL_setfuncs(_state, aoflib, 0); // Name the package (which pops the table) lua_setglobal(_state, "aoflagger"); } void LuaStrategy::Initialize() { luaL_openlibs(_state); loadaoflagger(); } void LuaStrategy::LoadFile(const char* filename) { check(luaL_loadfile(_state, filename)); check(lua_pcall(_state, 0, 0, 0)); } void LuaStrategy::LoadText(const std::string& data) { check(luaL_loadstring(_state, data.c_str())); check(lua_pcall(_state, 0, 0, 0)); } void LuaStrategy::RunPreamble(const std::vector& preamble) { for (const std::string& str : preamble) { luaL_loadstring(_state, str.c_str()); check(lua_pcall(_state, 0, 0, 0)); } } void LuaStrategy::Execute(TimeFrequencyData& tfData, TimeFrequencyMetaDataCPtr metaData, class ScriptData& scriptData, const std::string& executeFunctionName) { // store scriptData lua_pushstring(_state, "AOFlagger.ScriptData"); lua_pushlightuserdata(_state, &scriptData); // Set: registry["AOFlagger.ScriptData"] = scriptData lua_settable(_state, LUA_REGISTRYINDEX); lua_getglobal(_state, executeFunctionName.c_str()); if (lua_isnil(_state, -1)) { throw std::runtime_error( "The Lua script did not specify an execute function named " + executeFunctionName + "()."); } aoflagger_lua::Data* data = Tools::NewData(_state, tfData, metaData, _context); check(lua_pcall(_state, 1, 0, 0)); tfData = data->TFData(); for (aoflagger_lua::Data* data : _context.list) { if (!data->is_persistent()) data->clear(); } _context.list.clear(); } std::string LuaStrategy::GetTemplateScript() { return "function execute(data)\n" "\n" " data:clear_mask()\n" " -- Insert detection code here\n" "\n" "end\n" "\n" "function options()\n" " opts_main = {\n" " [\"baselines\"] = \"cross\"\n" " }\n" " return { [\"main\"] = opts_main }\n" "end\n"; } aoflagger-v3.5.1/lua/optionsfunction.cpp0000664000175000017500000002330115065216451016511 0ustar oleole#include "optionsfunction.h" #include extern "C" { #include #include } std::map OptionsFunction::GetOptions( lua_State* state, const Options& cmdLineOptions) { lua_getglobal(state, "options"); const int error = lua_pcall(state, 0, 1, 0); std::map optionMap; if (error) { lua_pop(state, 1); // pop error } else { // options function should have returned a table: if (!lua_istable(state, -1)) throw std::runtime_error("Function options() did not return a table"); // Iterate over the table. Keys are the name of the run, values // are itself a table with the options for that run. lua_pushnil(state); while (lua_next(state, -2) != 0) { // 'key' is at index -2 and 'value' is at index -1 // It is not allowed to change 'key' on the stack during traversal, and // lua_tostring() might change it. Therefore, we make a temp copy of the // key: lua_pushvalue(state, -2); const char* key = lua_tostring(state, -1); lua_pop(state, 1); if (key == nullptr) throw std::runtime_error( "Function options() returned a table with keys that were not " "convertable to a string"); if (!lua_istable(state, -1)) throw std::runtime_error( std::string("Invalid type of element '") + key + "' return by function options(): should be an option table"); const Options option = fillOptions(state, cmdLineOptions, std::string(key)); optionMap.emplace(key, option); // remove 'value'; keeps 'key' for next iteration lua_pop(state, 1); } } return optionMap; } std::string OptionsFunction::strOption(lua_State* state, const std::string& keyName, const std::string& runName) { const char* val = lua_tostring(state, -1); if (val == nullptr) throw std::runtime_error( "Option " + keyName + " for run name '" + runName + "' returned by options() was not convertable to a string"); return std::string(val); } bool OptionsFunction::boolOption(lua_State* state, const std::string& keyName, const std::string& runName) { if (!lua_isboolean(state, -1)) throw std::runtime_error( "Option " + keyName + " for run name '" + runName + "' returned by options() should be of type boolean"); return lua_toboolean(state, -1); } size_t OptionsFunction::uintOption(lua_State* state, const std::string& keyName, const std::string& runName) { if (!lua_isinteger(state, -1)) throw std::runtime_error( "Option " + keyName + " for run name '" + runName + "' returned by options() should be of type integer"); return lua_tointeger(state, -1); } std::vector OptionsFunction::uintListOption( lua_State* state, const std::string& keyName, const std::string& runName) { if (!lua_istable(state, -1)) throw std::runtime_error( "Option " + keyName + " for run name '" + runName + "' returned by options() should be a table of integers"); std::vector vals; lua_pushnil(state); while (lua_next(state, -2) != 0) { if (!lua_isinteger(state, -1)) throw std::runtime_error( "Option " + keyName + " for run name '" + runName + "' returned by options() should be a table of integers"); vals.emplace_back(lua_tointeger(state, -1)); lua_pop(state, 1); } return vals; } std::vector OptionsFunction::stringListOption( lua_State* state, const std::string& keyName, const std::string& runName) { if (!lua_istable(state, -1)) throw std::runtime_error( "Option " + keyName + " for run name '" + runName + "' returned by options() should be a table of strings"); std::vector vals; lua_pushnil(state); while (lua_next(state, -2) != 0) { if (!lua_isstring(state, -1)) throw std::runtime_error( "Option " + keyName + " for run name '" + runName + "' returned by options() should be a table of strings"); vals.emplace_back(lua_tostring(state, -1)); lua_pop(state, 1); } return vals; } Options OptionsFunction::fillOptions(lua_State* state, const Options& cmdLineOptions, const std::string& runName) { Options options; // Iterate over the table. Keys are the name of the run, values // are itself a table with the options for that run. lua_pushnil(state); while (lua_next(state, -2) != 0) { if (!lua_isstring(state, -2)) throw std::runtime_error("options(): Key in option table for run name '" + runName + "' was not convertable to a string"); const char* key = lua_tostring(state, -2); const std::string keyStr(key); if (keyStr == "bands") { std::vector list = uintListOption(state, keyStr, runName); options.bands = std::set(list.begin(), list.end()); } else if (keyStr == "baseline-integration") { options.baselineIntegration.enable = true; const std::string val = strOption(state, keyStr, runName); if (val == "count") options.baselineIntegration.mode = BaselineIntegrationMode::Count; else if (val == "average") options.baselineIntegration.mode = BaselineIntegrationMode::Average; else if (val == "average-abs") options.baselineIntegration.mode = BaselineIntegrationMode::AverageAbs; else if (val == "squared") options.baselineIntegration.mode = BaselineIntegrationMode::Squared; else if (val == "stddev") options.baselineIntegration.mode = BaselineIntegrationMode::Stddev; else throw std::runtime_error( "options(): Invalid setting '" + val + "' for option 'baseline-integration' returned"); } else if (keyStr == "baselines") { const std::string val = strOption(state, keyStr, runName); if (val == "all") options.baselineSelection = BaselineSelection::All; else if (val == "cross") options.baselineSelection = BaselineSelection::CrossCorrelations; else if (val == "auto") options.baselineSelection = BaselineSelection::AutoCorrelations; else throw std::runtime_error("options(): Invalid setting '" + val + "' for option 'baselines' returned"); } else if (keyStr == "chunk-size") { options.chunkSize = uintOption(state, keyStr, runName); } else if (keyStr == "column-name") { options.dataColumn = strOption(state, keyStr, runName); } else if (keyStr == "combine-spws") { options.combineSPWs = boolOption(state, keyStr, runName); } else if (keyStr == "execute-file") { options.executeFilename = strOption(state, keyStr, runName); } else if (keyStr == "execute-function") { options.executeFunctionName = strOption(state, keyStr, runName); } else if (keyStr == "fields") { std::vector list = uintListOption(state, keyStr, runName); options.fields = std::set(list.begin(), list.end()); } else if (keyStr == "files") { options.filenames = stringListOption(state, keyStr, runName); } else if (keyStr == "min‑aoflagger-version") { const std::string minVersion = strOption(state, keyStr, runName); const size_t dot = minVersion.find('.'); if (dot == minVersion.npos) throw std::runtime_error( "options(): Invalid version specified in option " "min-aoflagger-version: should be of the form major.minor"); int major = std::atoi(minVersion.substr(0, dot).c_str()), minor = std::atoi(minVersion.substr(dot + 1).c_str()); const bool tooOld = (AOFLAGGER_VERSION_MAJOR < major) || (AOFLAGGER_VERSION_MAJOR == major && AOFLAGGER_VERSION_MINOR < minor); if (tooOld) throw std::runtime_error( "AOFlagger is too old for this script: required: " + minVersion + ", this version: " + AOFLAGGER_VERSION_STR); } else if (keyStr == "quiet") { if (boolOption(state, keyStr, runName)) options.logVerbosity = Logger::QuietVerbosity; } else if (keyStr == "read-mode") { const std::string readMode = strOption(state, keyStr, runName); if (readMode == "direct") options.readMode = DirectReadMode; else if (readMode == "indirect") options.readMode = ReorderingReadMode; else if (readMode == "memory") options.readMode = MemoryReadMode; else if (readMode == "auto") options.readMode = AutoReadMode; else throw std::runtime_error( "options(): Invalid setting for option 'read-mode' returned"); } else if (keyStr == "read-uvws") { options.readUVW = boolOption(state, keyStr, runName); } else if (keyStr == "script-version") { options.scriptVersion = strOption(state, keyStr, runName); } else if (keyStr == "start-timestep") { options.startTimestep = uintOption(state, keyStr, runName); } else if (keyStr == "end-timestep") { options.endTimestep = uintOption(state, keyStr, runName); } else if (keyStr == "threads") { options.threadCount = uintOption(state, keyStr, runName); } else if (keyStr == "verbose") { // Option 'quiet' conflicts with verbose; quiet override verbose: if (boolOption(state, keyStr, runName) && !options.logVerbosity) options.logVerbosity = Logger::VerboseVerbosity; } else { Logger::Warn << "options(): Ignoring unknown key '" + keyStr + "'.\n"; } // remove 'value'; keeps 'key' for next iteration lua_pop(state, 1); } options.Override(cmdLineOptions); return options; } aoflagger-v3.5.1/lua/datawrapper.cpp0000664000175000017500000004271114752462134015573 0ustar oleole#include "datawrapper.h" #include "data.h" #include "tools.h" #include "../algorithms/restorechannelrange.h" using algorithms::RestoreChannelRange; int Data::clear_mask(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); data->TFData().SetNoMask(); return 0; } int Data::convert_to_complex(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const std::string reprStr = luaL_checklstring(L, 2, nullptr); try { enum TimeFrequencyData::ComplexRepresentation complexRepresentation; // PhasePart, AmplitudePart, RealPart, ImaginaryPart, ComplexParts if (reprStr == "phase") { complexRepresentation = TimeFrequencyData::PhasePart; } else if (reprStr == "amplitude") { complexRepresentation = TimeFrequencyData::AmplitudePart; } else if (reprStr == "real") { complexRepresentation = TimeFrequencyData::RealPart; } else if (reprStr == "imaginary") { complexRepresentation = TimeFrequencyData::ImaginaryPart; } else if (reprStr == "complex") { complexRepresentation = TimeFrequencyData::ComplexParts; } else { return luaL_error( L, "Unknown complex representation specified in convert_to_complex(): " "should be phase, amplitude, real, imaginary or complex"); } Tools::NewData(L, data->TFData().Make(complexRepresentation), data->MetaData(), data->GetContext()); return 1; } catch (std::exception& e) { return luaL_error( L, (std::string("convert_to_complex(): ") + e.what()).c_str()); } } int Data::convert_to_polarization(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const std::string polStr = luaL_checklstring(L, 2, nullptr); try { const aocommon::PolarizationEnum polarization = aocommon::Polarization::ParseString(polStr); Tools::NewData(L, data->TFData().Make(polarization), data->MetaData(), data->GetContext()); return 1; } catch (std::exception& e) { return luaL_error( L, (std::string("convert_to_polarization(): ") + e.what()).c_str()); } } int Data::copy(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); Tools::NewData(L, *data); return 1; } int Data::flag_zeros(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const Mask2DPtr mask(new Mask2D(*data->TFData().GetSingleMask())); const Image2DCPtr image = data->TFData().GetSingleImage(); for (unsigned y = 0; y < image->Height(); ++y) { for (unsigned x = 0; x < image->Width(); ++x) { if (image->Value(x, y) == 0.0) mask->SetValue(x, y, true); } } data->TFData().SetGlobalMask(mask); return 0; } int Data::flag_nans(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); TimeFrequencyData newData = data->TFData(); for (size_t p = 0; p != newData.PolarizationCount(); ++p) { TimeFrequencyData singlePol = newData.MakeFromPolarizationIndex(p); Mask2DPtr mask = Mask2D::MakePtr(*singlePol.GetSingleMask()); for (size_t i = 0; i != singlePol.ImageCount(); ++i) { const Image2DCPtr image = singlePol.GetImage(i); for (unsigned y = 0; y < image->Height(); ++y) { for (unsigned x = 0; x < image->Width(); ++x) { if (!std::isfinite(image->Value(x, y))) mask->SetValue(x, y, true); } } } singlePol.SetGlobalMask(std::move(mask)); newData.SetPolarizationData(p, std::move(singlePol)); } data->TFData() = newData; return 0; } int Data::get_antenna1_index(lua_State* L) { const aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); if (!data->MetaData()) luaL_error(L, "Can't call Data.antenna1_index(): no metadata available"); if (!data->MetaData()->HasAntenna1()) luaL_error(L, "Can't call Data.antenna1_index(): antenna1 not in metadata"); lua_pushinteger(L, data->MetaData()->Antenna1().id); return 1; } int Data::get_antenna1_name(lua_State* L) { const aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); if (!data->MetaData()) luaL_error(L, "Can't call Data.get_antenna1_name(): no metadata available"); if (!data->MetaData()->HasAntenna1()) luaL_error(L, "Can't call Data.get_antenna1_name(): antenna1 not in metadata"); lua_pushstring(L, data->MetaData()->Antenna1().name.c_str()); return 1; } int Data::get_antenna2_index(lua_State* L) { const aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); if (!data->MetaData()) luaL_error(L, "Can't call Data.antenna2_index(): no metadata available"); if (!data->MetaData()->HasAntenna2()) luaL_error(L, "Can't call Data.antenna2_index(): antenna1 not in metadata"); lua_pushinteger(L, data->MetaData()->Antenna2().id); return 1; } int Data::get_antenna2_name(lua_State* L) { const aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); if (!data->MetaData()) luaL_error(L, "Can't call Data.get_antenna2_name(): no metadata available"); if (!data->MetaData()->HasAntenna2()) luaL_error(L, "Can't call Data.get_antenna2_name(): antenna1 not in metadata"); lua_pushstring(L, data->MetaData()->Antenna2().name.c_str()); return 1; } int Data::get_baseline_angle(lua_State* L) { const aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); if (!data->MetaData()) luaL_error(L, "Can't call Data.get_baseline_angle(): no metadata available"); if (!data->MetaData()->HasBaseline()) luaL_error(L, "Can't call Data.get_baseline_angle(): basesline information " "not in metadata"); lua_pushnumber(L, data->MetaData()->Baseline().Angle()); return 1; } int Data::get_baseline_distance(lua_State* L) { const aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); if (!data->MetaData()) luaL_error( L, "Can't call Data.get_baseline_distance(): no metadata available"); if (!data->MetaData()->HasBaseline()) luaL_error(L, "Can't call Data.get_baseline_distance(): basesline information " "not in metadata"); lua_pushnumber(L, data->MetaData()->Baseline().Distance()); return 1; } int Data::get_baseline_vector(lua_State* L) { const aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); if (!data->MetaData()) luaL_error(L, "Can't call Data.get_baseline_vector(): no metadata available"); if (!data->MetaData()->HasBaseline()) luaL_error(L, "Can't call Data.get_baseline_vector(): basesline information " "not in metadata"); lua_newtable(L); lua_pushstring(L, "x"); lua_pushnumber(L, data->MetaData()->Baseline().DeltaX()); lua_settable(L, -3); lua_pushstring(L, "y"); lua_pushnumber(L, data->MetaData()->Baseline().DeltaY()); lua_settable(L, -3); lua_pushstring(L, "z"); lua_pushnumber(L, data->MetaData()->Baseline().DeltaZ()); lua_settable(L, -3); return 1; } int Data::get_complex_state(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); switch (data->TFData().ComplexRepresentation()) { case TimeFrequencyData::PhasePart: lua_pushstring(L, "phase"); break; case TimeFrequencyData::AmplitudePart: lua_pushstring(L, "amplitude"); break; case TimeFrequencyData::RealPart: lua_pushstring(L, "real"); break; case TimeFrequencyData::ImaginaryPart: lua_pushstring(L, "imaginary"); break; case TimeFrequencyData::ComplexParts: lua_pushstring(L, "complex"); break; } return 1; } int Data::get_frequencies(lua_State* L) { const aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); if (!data->MetaData()) luaL_error( L, "Error in call to Data.get_frequencies(): no metadata available"); if (!data->MetaData()->HasBand()) luaL_error( L, "Error in call to Data.get_frequencies(): no metadata available"); lua_newtable(L); const BandInfo& band = data->MetaData()->Band(); for (size_t i = 0; i != band.channels.size(); ++i) { const ChannelInfo& c = band.channels[i]; lua_pushnumber(L, c.frequencyHz); lua_rawseti(L, -2, i + 1); } return 1; } int Data::get_polarizations(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const std::vector pols = data->TFData().Polarizations(); lua_createtable(L, pols.size(), 0); for (size_t i = 0; i != pols.size(); ++i) { const aocommon::PolarizationEnum p = pols[i]; lua_pushstring(L, aocommon::Polarization::TypeToShortString(p).c_str()); lua_rawseti(L, -2, i + 1); } return 1; } int Data::get_times(lua_State* L) { const aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); if (!data->MetaData()) luaL_error(L, "Error in call to Data.get_times(): no metadata available"); if (!data->MetaData()->HasObservationTimes()) luaL_error(L, "Error in call to Data.get_times(): no metadata available"); lua_newtable(L); const std::vector& times = data->MetaData()->ObservationTimes(); for (size_t i = 0; i != times.size(); ++i) { lua_pushnumber(L, times[i]); lua_rawseti(L, -2, i + 1); } return 1; } int Data::has_metadata(lua_State* L) { const aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); bool hasMetaData; if (data->MetaData() != nullptr) { const TimeFrequencyMetaDataCPtr& md = data->MetaData(); hasMetaData = md->HasAntenna1() && md->HasAntenna2() && md->HasBand() && // Since there are no attributes of the field that can yet be // queried by Lua, for now Field is ignored: md->HasField() && md->HasObservationTimes(); } else { hasMetaData = false; } lua_pushboolean(L, hasMetaData); return 1; } int Data::invert_mask(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); for (size_t i = 0; i != data->TFData().MaskCount(); ++i) { Mask2DPtr mask = Mask2D::MakePtr(*data->TFData().GetMask(i)); mask->Invert(); data->TFData().SetMask(i, std::move(mask)); } return 0; } int Data::is_auto_correlation(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const bool isAuto = data->MetaData() && data->MetaData()->HasAntenna1() && data->MetaData()->HasAntenna2() && data->MetaData()->Antenna1().id == data->MetaData()->Antenna2().id; lua_pushboolean(L, isAuto); return 1; } int Data::is_complex(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const bool isComplex = data->TFData().ComplexRepresentation() == TimeFrequencyData::ComplexParts; lua_pushboolean(L, isComplex); return 1; } int Data::join_mask(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); aoflagger_lua::Data* other = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); try { data->TFData().JoinMask(other->TFData()); } catch (std::exception& e) { return luaL_error(L, (std::string("join_mask(): ") + e.what()).c_str()); } return 0; } int Data::set_mask(lua_State* L) { aoflagger_lua::Data *lhs = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")), *rhs = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); try { lhs->TFData().SetMask(rhs->TFData()); } catch (std::exception& e) { return luaL_error(L, (std::string("set_mask(): ") + e.what()).c_str()); } return 0; } int Data::set_mask_for_channel_range(lua_State* L) { aoflagger_lua::Data *lhs = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")), *rhs = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); double startMHz = luaL_checknumber(L, 3), endMHz = luaL_checknumber(L, 4); try { if (rhs->MetaData() != nullptr && rhs->MetaData()->HasBand()) RestoreChannelRange::Execute(lhs->TFData(), rhs->TFData(), *rhs->MetaData(), startMHz, endMHz); else throw std::runtime_error( "set_mask_for_channel_range(): No spectral band information " "available!"); } catch (std::exception& e) { return luaL_error( L, (std::string("set_mask_for_channel_range(): ") + e.what()).c_str()); } return 0; } int Data::set_masked_visibilities(lua_State* L) { aoflagger_lua::Data* lhs = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); aoflagger_lua::Data* rhs = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); if (rhs->TFData().ImageCount() != lhs->TFData().ImageCount() || rhs->TFData().PolarizationCount() != lhs->TFData().PolarizationCount()) { const std::string err = "set_masked_visibilities() was executed with inconsistent data types: " "right " "hand side had " + std::to_string(rhs->TFData().ImageCount()) + ", destination had " + std::to_string(lhs->TFData().ImageCount()); return luaL_error(L, err.c_str()); } for (size_t p = 0; p != rhs->TFData().PolarizationCount(); ++p) { TimeFrequencyData lPolData = lhs->TFData().MakeFromPolarizationIndex(p); const TimeFrequencyData rPolData = rhs->TFData().MakeFromPolarizationIndex(p); const Mask2DCPtr mask(lPolData.GetSingleMask()); for (size_t i = 0; i != rhs->TFData().ImageCount(); ++i) { const Image2DCPtr source(rhs->TFData().GetImage(i)); Image2DPtr dest(Image2D::MakePtr(*lhs->TFData().GetImage(i))); for (size_t y = 0; y != source->Height(); ++y) { for (size_t x = 0; x != source->Width(); ++x) { if (mask->Value(x, y)) dest->SetValue(x, y, source->Value(x, y)); } } lPolData.SetImage(i, std::move(dest)); } lhs->TFData().SetPolarizationData(p, std::move(lPolData)); } return 0; } int Data::set_polarization_data(lua_State* L) { aoflagger_lua::Data* lhs = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const std::string polStr = luaL_checklstring(L, 2, nullptr); aoflagger_lua::Data* rhs = reinterpret_cast( luaL_checkudata(L, 3, "AOFlaggerData")); try { const aocommon::PolarizationEnum polarization = aocommon::Polarization::ParseString(polStr); const size_t polIndex = lhs->TFData().GetPolarizationIndex(polarization); lhs->TFData().SetPolarizationData(polIndex, rhs->TFData()); } catch (std::exception& e) { return luaL_error( L, (std::string("set_polarization_data(): ") + e.what()).c_str()); } return 0; } int Data::set_visibilities(lua_State* L) { aoflagger_lua::Data* lhs = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); aoflagger_lua::Data* rhs = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); if (rhs->TFData().ImageCount() != lhs->TFData().ImageCount()) { const std::string err = "set_visibilities() was executed with inconsistent data types: right " "hand side had " + std::to_string(rhs->TFData().ImageCount()) + ", destination had " + std::to_string(lhs->TFData().ImageCount()); return luaL_error(L, err.c_str()); } for (size_t i = 0; i != rhs->TFData().ImageCount(); ++i) lhs->TFData().SetImage(i, rhs->TFData().GetImage(i)); return 0; } int Data::gc(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); data->~Data(); return 0; } int Data::sub(lua_State* L) { aoflagger_lua::Data* lhs = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); aoflagger_lua::Data* rhs = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); Tools::NewData(L, TimeFrequencyData::MakeFromDiff(lhs->TFData(), rhs->TFData()), lhs->MetaData(), lhs->GetContext()); return 1; } int Data::div(lua_State* L) { aoflagger_lua::Data* lhs = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); aoflagger_lua::Data* rhs = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); Tools::NewData(L, ElementWiseDivide(lhs->TFData(), rhs->TFData()), lhs->MetaData(), lhs->GetContext()); return 1; } aoflagger-v3.5.1/lua/scriptdata.h0000664000175000017500000000524614752462134015066 0ustar oleole#ifndef LUA_SCRIPTDATA_H #define LUA_SCRIPTDATA_H #include #include #include #include #include "../structures/timefrequencydata.h" class ScriptData { public: ScriptData(); ~ScriptData(); ScriptData(const ScriptData&) = delete; ScriptData& operator=(const ScriptData&) = delete; std::unique_ptr& GetBandpassFile() { return _bandpassFile; } std::mutex& BandpassMutex() { return _bandpassMutex; } std::unique_ptr& GetStatistics() { return _statistics; } void AddVisualization(TimeFrequencyData& data, const std::string& label, size_t sortingIndex) { if (_canVisualize) { if (data.PolarizationCount() == 1) { aocommon::PolarizationEnum p = data.GetPolarization(0); for (auto& v : _visualizationData) { if (std::get<0>(v) == label) { if (std::get<1>(v).HasPolarization(p)) { // Can't merge, continue search and // if not found add like normal } else { // Merge std::get<1>(v) = TimeFrequencyData::MakeFromPolarizationCombination( std::get<1>(v), data); return; } } } // Label not found, add _visualizationData.emplace_back(label, data, sortingIndex); } else { _visualizationData.emplace_back(label, data, sortingIndex); } } } size_t VisualizationCount() const { return _visualizationData.size(); } std::tuple GetVisualization( size_t index) const { return _visualizationData[index]; } void SetProgressListener(class ProgressListener& progressListener) { _progressListener = &progressListener; } ProgressListener* Progress() { return _progressListener; } void SetCanVisualize(bool canVisualize) { _canVisualize = canVisualize; } void SortVisualizations() { // Sort the visualizations on their sorting index std::sort(_visualizationData.begin(), _visualizationData.end(), [](const std::tuple& a, const std::tuple& b) { return std::get<2>(a) < std::get<2>(b); }); } void Combine(ScriptData&& other); private: class ProgressListener* _progressListener; std::unique_ptr _bandpassFile; std::mutex _bandpassMutex; bool _canVisualize; std::vector> _visualizationData; std::unique_ptr _statistics; }; #endif aoflagger-v3.5.1/lua/functions.h0000664000175000017500000000550315131147277014734 0ustar oleole#ifndef LUA_FUNCTIONS_H #define LUA_FUNCTIONS_H #include "data.h" #include class ScriptData; namespace aoflagger_lua { void apply_bandpass(Data& data, const std::string& filename, ScriptData& scriptData); void collect_statistics(const Data& dataAfter, const Data& dataBefore, ScriptData& scriptData); void copy_to_channel(Data& destination, const Data& source, size_t channel); void copy_to_frequency(Data& destination, const Data& source, double frequencyHz); Data downsample(const Data& data, size_t horizontalFactor, size_t verticalFactor); Data downsample_masked(const Data& data, size_t horizontalFactor, size_t verticalFactor); void upsample_image(const Data& input, Data& destination, size_t horizontalFactor, size_t verticalFactor); void upsample_mask(const Data& input, Data& destination, size_t horizontalFactor, size_t verticalFactor); void high_pass_filter(Data& data, size_t kernelWidth, size_t kernelHeight, double horizontalSigmaSquared, double verticalSigmaSquared); void low_pass_filter(Data& data, size_t kernelWidth, size_t kernelHeight, double horizontalSigmaSquared, double verticalSigmaSquared); Data norm(const Data& data); // TODO this function should collect the statistics and print // them later on (and be renamed). void print_polarization_statistics(const Data& data); void save_heat_map(const char* filename, const Data& data); void scale_invariant_rank_operator(Data& data, double level_horizontal, double level_vertical); void scale_invariant_rank_operator_masked(Data& data, const Data& missing, double level_horizontal, double level_vertical, double penalty); Data sqrt(const Data& data); void sumthreshold(Data& data, double hThresholdFactor, double vThresholdFactor, bool horizontal, bool vertical); void sumthreshold_masked(Data& data, const Data& missing, double hThresholdFactor, double vThresholdFactor, bool horizontal, bool vertical); void threshold_channel_rms(Data& data, double threshold, bool thresholdLowValues); void threshold_timestep_rms(Data& data, double threshold); Data trim_channels(const Data& data, size_t start_channel, size_t end_channel); Data trim_frequencies(const Data& data, double start_frequency, double end_frequency); void visualize(Data& data, const std::string& label, size_t sortingIndex, ScriptData& scriptData); } // namespace aoflagger_lua #endif aoflagger-v3.5.1/lua/luastrategy.h0000664000175000017500000000217314752462134015270 0ustar oleole#ifndef LUA_STRATEGY_H #define LUA_STRATEGY_H #include "data.h" #include "../structures/timefrequencymetadata.h" extern "C" { #include #include #include } #include #include class LuaStrategy { public: LuaStrategy(); ~LuaStrategy() { clear(); } LuaStrategy(LuaStrategy&& source); LuaStrategy& operator=(LuaStrategy&& source); LuaStrategy(const LuaStrategy&) = delete; LuaStrategy& operator=(const LuaStrategy&) = delete; void Initialize(); void LoadFile(const char* filename); void LoadText(const std::string& data); void Execute(class TimeFrequencyData& tfData, TimeFrequencyMetaDataCPtr metaData, class ScriptData& scriptData, const std::string& executeFunctionName); void RunPreamble(const std::vector& preamble); lua_State* State() { return _state; } static std::string GetTemplateScript(); private: void loadaoflagger(); static void check(lua_State* state, int error); void check(int error) { check(_state, error); } void clear(); aoflagger_lua::Data::Context _context; lua_State* _state; }; #endif aoflagger-v3.5.1/lua/luathreadgroup.h0000664000175000017500000000234414752462134015752 0ustar oleole#ifndef LUA_THREAD_GROUP_H #define LUA_THREAD_GROUP_H #include "luastrategy.h" #include class LuaThreadGroup { public: explicit LuaThreadGroup(size_t nThreads) : _strategies(nThreads) {} void LoadFile(const char* filename) { for (LuaStrategy& s : _strategies) { s.Initialize(); s.LoadFile(filename); } } void LoadText(const std::string& text) { for (LuaStrategy& s : _strategies) { s.Initialize(); s.LoadText(text); } } void Execute(size_t threadIndex, class TimeFrequencyData& tfData, const TimeFrequencyMetaDataCPtr& metaData, class ScriptData& scriptData, const std::string& executeFunctionName) { _strategies[threadIndex].Execute(tfData, metaData, scriptData, executeFunctionName); } void RunPreamble(const std::vector& preamble) { for (LuaStrategy& s : _strategies) s.RunPreamble(preamble); } size_t NThreads() const { return _strategies.size(); } const LuaStrategy& GetThread(size_t index) const { return _strategies[index]; } LuaStrategy& GetThread(size_t index) { return _strategies[index]; } private: std::vector _strategies; }; #endif aoflagger-v3.5.1/lua/datawrapper.h0000664000175000017500000000272014752462134015234 0ustar oleole#ifndef LUA_DATA_WRAPPER_H #define LUA_DATA_WRAPPER_H extern "C" { #include #include #include } #include class Data { public: static int clear_mask(lua_State* L); static int convert_to_complex(lua_State* L); static int convert_to_polarization(lua_State* L); static int copy(lua_State* L); static int flag_zeros(lua_State* L); static int flag_nans(lua_State* L); static int get_antenna1_index(lua_State* L); static int get_antenna1_name(lua_State* L); static int get_antenna2_index(lua_State* L); static int get_antenna2_name(lua_State* L); static int get_baseline_angle(lua_State* L); static int get_baseline_distance(lua_State* L); static int get_baseline_vector(lua_State* L); static int get_complex_state(lua_State* L); static int get_frequencies(lua_State* L); static int get_polarizations(lua_State* L); static int get_times(lua_State* L); static int has_metadata(lua_State* L); static int invert_mask(lua_State* L); static int is_auto_correlation(lua_State* L); static int is_complex(lua_State* L); static int join_mask(lua_State* L); static int set_mask(lua_State* L); static int set_mask_for_channel_range(lua_State* L); static int set_masked_visibilities(lua_State* L); static int set_visibilities(lua_State* L); static int set_polarization_data(lua_State* L); static int gc(lua_State* L); static int div(lua_State* L); static int sub(lua_State* L); private: }; #endif aoflagger-v3.5.1/lua/default-strategy.h0000664000175000017500000000102314752462134016201 0ustar oleole#ifndef DATA_DEFAULT_STRATEGY_H #define DATA_DEFAULT_STRATEGY_H // The file default-strategy.cpp is generated with a run of 'xxd -i', to make it // possible to include the default strategy literally in the flagger. I do this // because not all platforms install data files in a standardized place when // using installing in a prefix. This way, at least the default strategy is // always available. extern unsigned char data_strategies_generic_default_lua[]; extern unsigned int data_strategies_generic_default_lua_len; #endif aoflagger-v3.5.1/lua/functions.cpp0000664000175000017500000004405415131147277015273 0ustar oleole#include "functions.h" #include "scriptdata.h" #include "../algorithms/applybandpass.h" #include "../algorithms/highpassfilter.h" #include "../algorithms/medianwindow.h" #include "../algorithms/resampling.h" #include "../algorithms/siroperator.h" #include "../algorithms/thresholdconfig.h" #include "../structures/image2d.h" #include "../structures/samplerow.h" #include "../structures/timefrequencydata.h" #ifdef HAVE_GTKMM #include "../rfigui/maskedheatmap.h" #endif #include "../algorithms/polarizationstatistics.h" #include "../algorithms/thresholdtools.h" #include "../quality/statisticscollection.h" #include #include using algorithms::ApplyBandpass; using algorithms::HighPassFilter; using algorithms::MedianWindow; using algorithms::PolarizationStatistics; using algorithms::SIROperator; using algorithms::ThresholdConfig; using algorithms::ThresholdTools; namespace aoflagger_lua { void apply_bandpass(Data& data, const std::string& filename, ScriptData& scriptData) { std::unique_ptr& bpFile = scriptData.GetBandpassFile(); { const std::lock_guard lock(scriptData.BandpassMutex()); if (bpFile == nullptr) { bpFile.reset(new BandpassFile(filename)); } } ApplyBandpass::Apply(data.TFData(), *bpFile, data.MetaData()->Antenna1().name, data.MetaData()->Antenna2().name); } void collect_statistics(const Data& dataAfter, const Data& dataBefore, ScriptData& scriptData) { std::unique_ptr& statistics(scriptData.GetStatistics()); const size_t polarizationCount = dataAfter.TFData().PolarizationCount(); if (dataBefore.TFData().PolarizationCount() != polarizationCount) throw std::runtime_error( "Before and after have different nr of polarizations in call to " "collect_statistics()"); if (dataAfter.TFData().ComplexRepresentation() != TimeFrequencyData::ComplexParts) throw std::runtime_error( "collect_statistics(): statistics can only be collected for complex " "data, first parameter is not complex"); if (dataBefore.TFData().ComplexRepresentation() != TimeFrequencyData::ComplexParts) throw std::runtime_error( "collect_statistics(): statistics can only be collected for complex " "data, second parameter is not complex"); if (!dataBefore.MetaData()) throw std::runtime_error("collect_statistics(): missing metadata"); if (!dataBefore.MetaData()->HasBand()) throw std::runtime_error("collect_statistics(): missing band metadata"); if (!statistics) statistics.reset(new StatisticsCollection(polarizationCount)); const size_t bandIndex = dataBefore.MetaData()->Band().windowIndex; if (!statistics->HasBand(bandIndex)) { std::vector channels(dataBefore.MetaData()->Band().channels.size()); for (size_t i = 0; i != channels.size(); ++i) channels[i] = dataBefore.MetaData()->Band().channels[i].frequencyHz; statistics->InitializeBand(bandIndex, channels.data(), channels.size()); } const bool useEmpty = (dataBefore.TFData().MaskCount() == 0 || dataAfter.TFData().MaskCount() == 0); Mask2DPtr emptyMask; if (useEmpty) { // TODO we can avoid this allocation when StatisticsCollection::AddImage() // would support a call without a 2nd mask emptyMask = Mask2D::CreateSetMaskPtr( dataBefore.TFData().ImageWidth(), dataBefore.TFData().ImageHeight()); } if (!dataAfter.MetaData()->HasAntenna1() || !dataAfter.MetaData()->HasAntenna2() || !dataAfter.MetaData()->HasObservationTimes()) throw std::runtime_error( "collect_statistics(): can't collect statistics for sets without " "metadata (antenna info, time info)"); size_t antenna1 = dataAfter.MetaData()->Antenna1().id, antenna2 = dataAfter.MetaData()->Antenna2().id; const std::vector& times = dataAfter.MetaData()->ObservationTimes(); for (size_t polarization = 0; polarization != polarizationCount; ++polarization) { TimeFrequencyData polDataBefore = dataBefore.TFData().MakeFromPolarizationIndex( polarization), polDataAfter = dataAfter.TFData().MakeFromPolarizationIndex( polarization); Mask2DCPtr beforeMask, afterMask; if (dataBefore.TFData().MaskCount() == 0) beforeMask = emptyMask; else beforeMask = polDataBefore.GetSingleMask(); if (dataAfter.TFData().MaskCount() == 0) afterMask = emptyMask; else afterMask = polDataAfter.GetSingleMask(); statistics->AddImage(antenna1, antenna2, ×[0], bandIndex, polarization, polDataBefore.GetRealPart(), polDataBefore.GetImaginaryPart(), afterMask, beforeMask); } } void copy_to_channel(Data& destination, const Data& source, size_t channel) { if (channel >= destination.TFData().ImageHeight()) throw std::runtime_error( "copy_to_channel(): channel parameter is outside the band"); destination.TFData().CopyFrom(source.TFData(), 0, channel); } void copy_to_frequency(Data& destination, const Data& source, double frequencyHz) { if (destination.MetaData() == nullptr || !destination.MetaData()->HasBand()) throw std::runtime_error( "copy_to_frequency(): no frequency meta data available in data object"); const BandInfo& band = destination.MetaData()->Band(); ChannelInfo channel; channel.frequencyHz = frequencyHz; std::vector::const_iterator iter; if (band.channels.begin() > band.channels.end()) { iter = std::lower_bound( band.channels.begin(), band.channels.end(), channel, [](const ChannelInfo& lhs, const ChannelInfo& rhs) -> bool { return lhs.frequencyHz > rhs.frequencyHz; }); } else { iter = std::lower_bound( band.channels.begin(), band.channels.end(), channel, [](const ChannelInfo& lhs, const ChannelInfo& rhs) -> bool { return lhs.frequencyHz < rhs.frequencyHz; }); } const size_t channelIndex = iter - band.channels.begin(); copy_to_channel(destination, source, channelIndex); } void upsample_image(const Data& input, Data& destination, size_t horizontalFactor, size_t verticalFactor) { algorithms::upsample_image(input.TFData(), destination.TFData(), horizontalFactor, verticalFactor); } void upsample_mask(const Data& input, Data& destination, size_t horizontalFactor, size_t verticalFactor) { algorithms::upsample_mask(input.TFData(), destination.TFData(), horizontalFactor, verticalFactor); } void low_pass_filter(Data& data, size_t kernelWidth, size_t kernelHeight, double horizontalSigmaSquared, double verticalSigmaSquared) { if (data.TFData().PolarizationCount() != 1) throw std::runtime_error("High-pass filtering needs single polarization"); HighPassFilter filter; filter.SetHWindowSize(kernelWidth); filter.SetVWindowSize(kernelHeight); filter.SetHKernelSigmaSq(horizontalSigmaSquared); filter.SetVKernelSigmaSq(verticalSigmaSquared); const Mask2DCPtr mask = data.TFData().GetSingleMask(); const size_t imageCount = data.TFData().ImageCount(); for (size_t i = 0; i < imageCount; ++i) data.TFData().SetImage( i, filter.ApplyLowPass(data.TFData().GetImage(i), mask)); } void high_pass_filter(Data& data, size_t kernelWidth, size_t kernelHeight, double horizontalSigmaSquared, double verticalSigmaSquared) { if (data.TFData().PolarizationCount() != 1) throw std::runtime_error("High-pass filtering needs single polarization"); HighPassFilter filter; filter.SetHWindowSize(kernelWidth); filter.SetVWindowSize(kernelHeight); filter.SetHKernelSigmaSq(horizontalSigmaSquared); filter.SetVKernelSigmaSq(verticalSigmaSquared); const Mask2DCPtr mask = data.TFData().GetSingleMask(); const size_t imageCount = data.TFData().ImageCount(); for (size_t i = 0; i < imageCount; ++i) data.TFData().SetImage( i, filter.ApplyHighPass(data.TFData().GetImage(i), mask)); } Data norm(const Data& data) { return Data(ElementWiseNorm(data.TFData()), data.MetaData(), data.GetContext()); } void save_heat_map(const char* filename, const Data& data) { #ifdef HAVE_GTKMM const TimeFrequencyData tfData = data.TFData(); MaskedHeatMap plot; plot.SetImage( std::unique_ptr(new PlotImage(tfData.GetSingleImage()))); plot.SetAlternativeMask(tfData.GetSingleMask()); plot.SaveByExtension(filename, 800, 500); #else throw std::runtime_error("Compiled without GTKMM -- can not save heat map"); #endif } void print_polarization_statistics(const Data& data) { PolarizationStatistics statistics; statistics.Add(data.TFData()); statistics.Report(); } void scale_invariant_rank_operator(Data& data, double level_horizontal, double level_vertical) { if (!data.TFData().IsEmpty()) { const Mask2DPtr mask(new Mask2D(*data.TFData().GetSingleMask())); SIROperator::OperateHorizontally(*mask, level_horizontal); SIROperator::OperateVertically(*mask, level_vertical); data.TFData().SetGlobalMask(mask); } } void scale_invariant_rank_operator_masked(Data& data, const Data& missing, double level_horizontal, double level_vertical, double penalty) { if (!data.TFData().IsEmpty()) { const Mask2DPtr mask(new Mask2D(*data.TFData().GetSingleMask())); const Mask2DCPtr missingMask = missing.TFData().GetSingleMask(); SIROperator::OperateHorizontallyMissing(*mask, *missingMask, level_horizontal, penalty); SIROperator::OperateVerticallyMissing(*mask, *missingMask, level_vertical, penalty); data.TFData().SetGlobalMask(mask); } } Data sqrt(const Data& data) { return Data(ElementWiseSqrt(data.TFData()), data.MetaData(), data.GetContext()); } Data downsample(const Data& data, size_t horizontalFactor, size_t verticalFactor) { TimeFrequencyData timeFrequencyData = data.TFData(); const size_t imageCount = timeFrequencyData.ImageCount(); const size_t maskCount = timeFrequencyData.MaskCount(); if (horizontalFactor > 1) { for (size_t i = 0; i < imageCount; ++i) { const Image2DPtr newImage(new Image2D( timeFrequencyData.GetImage(i)->ShrinkHorizontally(horizontalFactor))); timeFrequencyData.SetImage(i, newImage); } for (size_t i = 0; i < maskCount; ++i) { const Mask2DPtr newMask(new Mask2D( timeFrequencyData.GetMask(i)->ShrinkHorizontally(horizontalFactor))); timeFrequencyData.SetMask(i, newMask); } } if (verticalFactor > 1) { for (size_t i = 0; i < imageCount; ++i) { const Image2DPtr newImage(new Image2D( timeFrequencyData.GetImage(i)->ShrinkVertically(verticalFactor))); timeFrequencyData.SetImage(i, newImage); } for (size_t i = 0; i < maskCount; ++i) { const Mask2DPtr newMask(new Mask2D( timeFrequencyData.GetMask(i)->ShrinkVertically(verticalFactor))); timeFrequencyData.SetMask(i, newMask); } } return Data(timeFrequencyData, data.MetaData(), data.GetContext()); } Data downsample_masked(const Data& data, size_t horizontalFactor, size_t verticalFactor) { TimeFrequencyData timeFrequencyData = data.TFData(); TimeFrequencyMetaDataPtr metaData; if (data.MetaData()) { metaData.reset(new TimeFrequencyMetaData(*data.MetaData())); algorithms::downsample_masked(timeFrequencyData, metaData.get(), horizontalFactor, verticalFactor); } else { algorithms::downsample_masked(timeFrequencyData, nullptr, horizontalFactor, verticalFactor); } return Data(timeFrequencyData, metaData, data.GetContext()); } static void sumthreshold_generic(Data& data, const Data* missing, double hThresholdFactor, double vThresholdFactor, bool horizontal, bool vertical) { ThresholdConfig thresholdConfig; thresholdConfig.InitializeLengthsDefault(); thresholdConfig.InitializeThresholdsFromFirstThreshold( 6.0L, ThresholdConfig::Rayleigh); if (!horizontal) thresholdConfig.RemoveHorizontalOperations(); if (!vertical) thresholdConfig.RemoveVerticalOperations(); if (data.TFData().PolarizationCount() != 1) throw std::runtime_error("Input data in sum_threshold has wrong format"); const Mask2DPtr mask(new Mask2D(*data.TFData().GetSingleMask())); const Image2DCPtr image = data.TFData().GetSingleImage(); if (missing != nullptr) { const Mask2DCPtr missingMask = missing->TFData().GetSingleMask(); thresholdConfig.ExecuteWithMissing(image.get(), mask.get(), missingMask.get(), false, hThresholdFactor, vThresholdFactor); } else { thresholdConfig.Execute(image.get(), mask.get(), false, hThresholdFactor, vThresholdFactor); } data.TFData().SetGlobalMask(mask); } void sumthreshold(Data& data, double hThresholdFactor, double vThresholdFactor, bool horizontal, bool vertical) { sumthreshold_generic(data, nullptr, hThresholdFactor, vThresholdFactor, horizontal, vertical); } void sumthreshold_masked(Data& data, const Data& missing, double hThresholdFactor, double vThresholdFactor, bool horizontal, bool vertical) { sumthreshold_generic(data, &missing, hThresholdFactor, vThresholdFactor, horizontal, vertical); } void threshold_channel_rms(Data& data, double threshold, bool thresholdLowValues) { const Image2DCPtr image(data.TFData().GetSingleImage()); SampleRow channels = SampleRow::MakeEmpty(image->Height()); Mask2DPtr mask(new Mask2D(*data.TFData().GetSingleMask())); for (size_t y = 0; y < image->Height(); ++y) { const SampleRow row = SampleRow::MakeFromRowWithMissings(image.get(), mask.get(), y); channels.SetValue(y, row.RMSWithMissings()); } bool change; do { const num_t median = channels.MedianWithMissings(); const num_t stddev = channels.StdDevWithMissings(median); change = false; const double effectiveThreshold = threshold * stddev; for (size_t y = 0; y < channels.Size(); ++y) { if (!channels.ValueIsMissing(y) && (channels.Value(y) - median > effectiveThreshold || (thresholdLowValues && median - channels.Value(y) > effectiveThreshold))) { mask->SetAllHorizontally(y); channels.SetValueMissing(y); change = true; } } } while (change); data.TFData().SetGlobalMask(std::move(mask)); } void threshold_timestep_rms(Data& data, double threshold) { if (!data.TFData().IsEmpty()) { const Image2DCPtr image = data.TFData().GetSingleImage(); SampleRow timesteps = SampleRow::MakeEmpty(image->Width()); Mask2DPtr mask(new Mask2D(*data.TFData().GetSingleMask())); for (size_t x = 0; x < image->Width(); ++x) { const SampleRow row = SampleRow::MakeFromColumnWithMissings(image.get(), mask.get(), x); timesteps.SetValue(x, row.RMSWithMissings()); } bool change; MedianWindow::SubtractMedian(timesteps, 511); do { const num_t median = 0.0; const num_t stddev = timesteps.StdDevWithMissings(0.0); change = false; for (size_t x = 0; x < timesteps.Size(); ++x) { if (!timesteps.ValueIsMissing(x) && (timesteps.Value(x) - median > stddev * threshold || median - timesteps.Value(x) > stddev * threshold)) { mask->SetAllVertically(x); timesteps.SetValueMissing(x); change = true; } } } while (change); data.TFData().SetGlobalMask(std::move(mask)); } } Data trim_channels(const Data& data, size_t start_channel, size_t end_channel) { if (start_channel > data.TFData().ImageHeight()) throw std::runtime_error("trim_channels(): Invalid start channel"); if (end_channel > data.TFData().ImageHeight()) throw std::runtime_error("trim_channels(): Invalid end channel"); if (start_channel >= end_channel) throw std::runtime_error("trim_channels(): Invalid range (start >= end)"); TimeFrequencyData trimmedData = data.TFData(); trimmedData.Trim(0, start_channel, trimmedData.ImageWidth(), end_channel); if (data.MetaData()) { const TimeFrequencyMetaDataPtr metaData( new TimeFrequencyMetaData(*data.MetaData())); if (metaData->HasBand()) { // Correct the band data BandInfo band = metaData->Band(); band.channels.assign( data.MetaData()->Band().channels.begin() + start_channel, data.MetaData()->Band().channels.begin() + end_channel); metaData->SetBand(band); } return Data(std::move(trimmedData), metaData, data.GetContext()); } else { return Data(std::move(trimmedData), nullptr, data.GetContext()); } } Data trim_frequencies(const Data& data, double start_frequency, double end_frequency) { if (start_frequency >= end_frequency) throw std::runtime_error( "trim_frequencies(): Invalid range (start >= end)"); if (data.MetaData() != nullptr && data.MetaData()->HasBand()) { const std::pair channelRange = data.MetaData()->Band().GetChannelRange(start_frequency, end_frequency); return trim_channels(data, channelRange.first, channelRange.second); } else { throw std::runtime_error( "trim_frequency(): No spectral band information available!"); } } void visualize(Data& data, const std::string& label, size_t sortingIndex, ScriptData& scriptData) { scriptData.AddVisualization(data.TFData(), label, sortingIndex); } } // namespace aoflagger_lua aoflagger-v3.5.1/lua/scriptdata.cpp0000664000175000017500000000152214752462134015412 0ustar oleole#include "scriptdata.h" #include "../algorithms/bandpassfile.h" #include "../quality/statisticscollection.h" ScriptData::ScriptData() : _progressListener(nullptr), _bandpassFile(), _bandpassMutex(), _canVisualize(false), _visualizationData(), _statistics() {} ScriptData::~ScriptData() {} void ScriptData::Combine(ScriptData&& other) { if (!_bandpassFile) _bandpassFile = std::move(other._bandpassFile); if (_canVisualize) { _visualizationData.insert(_visualizationData.end(), other._visualizationData.begin(), other._visualizationData.end()); other._visualizationData.clear(); } if (other._statistics) { if (_statistics) _statistics->Add(*other._statistics); else _statistics = std::move(other._statistics); } } aoflagger-v3.5.1/lua/optionsfunction.h0000664000175000017500000000254514752462134016170 0ustar oleole#ifndef LUA_OPTIONS_FUNCTION_H #define LUA_OPTIONS_FUNCTION_H #include #include #include #include "../aoluarunner/options.h" extern "C" { #include } class OptionsFunction { public: // Returns a map of options. The map keys are the name of the runs. static std::map GetOptions( lua_State* state, const Options& cmdLineOptions); private: static Options fillOptions(lua_State* state, const Options& cmdLineOptions, const std::string& runName); static bool boolOption(lua_State* state, const std::string& keyName, const std::string& runName); static size_t uintOption(lua_State* state, const std::string& keyName, const std::string& runName); static std::vector uintListOption(lua_State* state, const std::string& keyName, const std::string& runName); static std::vector stringListOption(lua_State* state, const std::string& keyName, const std::string& runName); static std::string strOption(lua_State* state, const std::string& keyName, const std::string& runName); }; #endif aoflagger-v3.5.1/lua/default-strategy.cpp0000664000175000017500000011230214752462134016537 0ustar oleoleunsigned char data_strategies_generic_default_lua[] = { 0x2d, 0x2d, 0x5b, 0x5b, 0x0a, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x74, 0x68, 0x65, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x41, 0x4f, 0x46, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x20, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x2c, 0x20, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x32, 0x30, 0x32, 0x31, 0x2d, 0x30, 0x33, 0x2d, 0x33, 0x30, 0x0a, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x3a, 0x20, 0x41, 0x6e, 0x64, 0x72, 0xc3, 0xa9, 0x20, 0x4f, 0x66, 0x66, 0x72, 0x69, 0x6e, 0x67, 0x61, 0x0a, 0x0a, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x20, 0x69, 0x73, 0x20, 0x6d, 0x61, 0x64, 0x65, 0x20, 0x61, 0x73, 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x20, 0x2f, 0x20, 0x65, 0x61, 0x73, 0x79, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x77, 0x65, 0x61, 0x6b, 0x20, 0x61, 0x73, 0x20, 0x70, 0x6f, 0x73, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x2c, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6d, 0x6f, 0x73, 0x74, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6e, 0x74, 0x0a, 0x27, 0x74, 0x77, 0x65, 0x61, 0x6b, 0x69, 0x6e, 0x67, 0x27, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x20, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x20, 0x61, 0x73, 0x20, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x20, 0x61, 0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x20, 0x6f, 0x66, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x27, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x27, 0x2e, 0x0a, 0x5d, 0x5d, 0x0a, 0x0a, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x5f, 0x6d, 0x69, 0x6e, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x33, 0x2e, 0x30, 0x22, 0x29, 0x0a, 0x0a, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x28, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x29, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x0a, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x57, 0x68, 0x61, 0x74, 0x20, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x3f, 0x20, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x3a, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x28, 0x29, 0x20, 0x28, 0x3d, 0x61, 0x6c, 0x6c, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x61, 0x72, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x64, 0x61, 0x74, 0x61, 0x29, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x65, 0x2e, 0x67, 0x2e, 0x3a, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x7b, 0x20, 0x27, 0x58, 0x59, 0x27, 0x2c, 0x20, 0x27, 0x59, 0x58, 0x27, 0x20, 0x7d, 0x20, 0x74, 0x6f, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x20, 0x6f, 0x6e, 0x6c, 0x79, 0x20, 0x58, 0x59, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x59, 0x58, 0x2c, 0x20, 0x6f, 0x72, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x7b, 0x20, 0x27, 0x49, 0x27, 0x2c, 0x20, 0x27, 0x51, 0x27, 0x20, 0x7d, 0x20, 0x74, 0x6f, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x20, 0x6f, 0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x20, 0x53, 0x74, 0x6f, 0x6b, 0x65, 0x73, 0x20, 0x49, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x51, 0x0a, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x3d, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x28, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x20, 0x3d, 0x20, 0x31, 0x2e, 0x30, 0x20, 0x2d, 0x2d, 0x20, 0x6c, 0x6f, 0x77, 0x65, 0x72, 0x20, 0x6d, 0x65, 0x61, 0x6e, 0x73, 0x20, 0x6d, 0x6f, 0x72, 0x65, 0x20, 0x73, 0x65, 0x6e, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x20, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x48, 0x6f, 0x77, 0x20, 0x74, 0x6f, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2c, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x61, 0x72, 0x65, 0x3a, 0x20, 0x70, 0x68, 0x61, 0x73, 0x65, 0x2c, 0x20, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x74, 0x75, 0x64, 0x65, 0x2c, 0x20, 0x72, 0x65, 0x61, 0x6c, 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x2c, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x4d, 0x61, 0x79, 0x20, 0x68, 0x61, 0x76, 0x65, 0x20, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x70, 0x65, 0x72, 0x66, 0x6f, 0x72, 0x6d, 0x20, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x0a, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x72, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x22, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x74, 0x75, 0x64, 0x65, 0x22, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x33, 0x20, 0x2d, 0x2d, 0x20, 0x68, 0x6f, 0x77, 0x20, 0x6d, 0x61, 0x6e, 0x79, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x70, 0x65, 0x72, 0x66, 0x6f, 0x72, 0x6d, 0x3f, 0x0a, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x74, 0x65, 0x70, 0x20, 0x3d, 0x20, 0x32, 0x2e, 0x30, 0x20, 0x2d, 0x2d, 0x20, 0x48, 0x6f, 0x77, 0x20, 0x6d, 0x75, 0x63, 0x68, 0x20, 0x74, 0x6f, 0x20, 0x69, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73, 0x65, 0x6e, 0x73, 0x69, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x20, 0x65, 0x61, 0x63, 0x68, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x3f, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x49, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x20, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x20, 0x69, 0x73, 0x20, 0x74, 0x72, 0x75, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x20, 0x77, 0x69, 0x6c, 0x6c, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, 0x20, 0x65, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x28, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x29, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x61, 0x73, 0x20, 0x62, 0x61, 0x64, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x20, 0x49, 0x74, 0x20, 0x77, 0x69, 0x6c, 0x6c, 0x20, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6f, 0x6e, 0x2d, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x64, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2c, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x6d, 0x61, 0x6b, 0x65, 0x20, 0x73, 0x75, 0x72, 0x65, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x65, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x77, 0x69, 0x6c, 0x6c, 0x20, 0x73, 0x74, 0x61, 0x79, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x64, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x2e, 0x20, 0x49, 0x66, 0x20, 0x73, 0x65, 0x74, 0x20, 0x74, 0x6f, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x2c, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x64, 0x2e, 0x0a, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x75, 0x73, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x3d, 0x20, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x72, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x20, 0x3d, 0x20, 0x31, 0x2e, 0x30, 0x20, 0x2d, 0x2d, 0x20, 0x41, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x22, 0x65, 0x78, 0x74, 0x72, 0x61, 0x22, 0x20, 0x73, 0x6d, 0x6f, 0x6f, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x20, 0x69, 0x6e, 0x20, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x20, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x0a, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x20, 0x3d, 0x20, 0x31, 0x2e, 0x30, 0x20, 0x2d, 0x2d, 0x20, 0x64, 0x65, 0x63, 0x72, 0x65, 0x61, 0x73, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x6d, 0x61, 0x6b, 0x65, 0x73, 0x20, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x52, 0x46, 0x49, 0x20, 0x6d, 0x6f, 0x72, 0x65, 0x20, 0x61, 0x67, 0x67, 0x72, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x0a, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x45, 0x6e, 0x64, 0x20, 0x6f, 0x66, 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x0a, 0x0a, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x69, 0x6e, 0x70, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x3d, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x28, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x69, 0x66, 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x75, 0x73, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x63, 0x6c, 0x65, 0x61, 0x72, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x28, 0x29, 0x0a, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x46, 0x6f, 0x72, 0x20, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x73, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x2e, 0x20, 0x4e, 0x6f, 0x74, 0x65, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x64, 0x6f, 0x6e, 0x65, 0x20, 0x61, 0x66, 0x74, 0x65, 0x72, 0x20, 0x63, 0x6c, 0x65, 0x61, 0x72, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x28, 0x29, 0x2c, 0x0a, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x73, 0x6f, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x20, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x0a, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x63, 0x6f, 0x70, 0x79, 0x5f, 0x6f, 0x66, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x3d, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x63, 0x6f, 0x70, 0x79, 0x28, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x69, 0x70, 0x6f, 0x6c, 0x2c, 0x20, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x69, 0x70, 0x61, 0x69, 0x72, 0x73, 0x28, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x29, 0x20, 0x64, 0x6f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x70, 0x6f, 0x6c, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x5f, 0x74, 0x6f, 0x5f, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x70, 0x79, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x5f, 0x2c, 0x20, 0x72, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x69, 0x70, 0x61, 0x69, 0x72, 0x73, 0x28, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x72, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x29, 0x20, 0x64, 0x6f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x70, 0x6f, 0x6c, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x5f, 0x74, 0x6f, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x28, 0x72, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x70, 0x79, 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x63, 0x6f, 0x70, 0x79, 0x28, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x69, 0x20, 0x3d, 0x20, 0x31, 0x2c, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x20, 0x2d, 0x20, 0x31, 0x20, 0x64, 0x6f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x74, 0x65, 0x70, 0x20, 0x5e, 0x20, 0x28, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x20, 0x2d, 0x20, 0x69, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x73, 0x75, 0x6d, 0x74, 0x68, 0x72, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x20, 0x2a, 0x20, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x75, 0x73, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x73, 0x75, 0x6d, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x65, 0x64, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x70, 0x79, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, 0x6d, 0x74, 0x68, 0x72, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, 0x6d, 0x74, 0x68, 0x72, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x20, 0x2a, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x75, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x73, 0x75, 0x6d, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x73, 0x75, 0x6d, 0x74, 0x68, 0x72, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x2c, 0x20, 0x73, 0x75, 0x6d, 0x74, 0x68, 0x72, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x20, 0x2a, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x44, 0x6f, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x65, 0x70, 0x20, 0x26, 0x20, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x63, 0x68, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x63, 0x6f, 0x70, 0x79, 0x28, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x65, 0x70, 0x5f, 0x72, 0x6d, 0x73, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x33, 0x2e, 0x35, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x72, 0x6d, 0x73, 0x28, 0x63, 0x68, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x33, 0x2e, 0x30, 0x20, 0x2a, 0x20, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x6a, 0x6f, 0x69, 0x6e, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x28, 0x63, 0x68, 0x64, 0x61, 0x74, 0x61, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x48, 0x69, 0x67, 0x68, 0x20, 0x70, 0x61, 0x73, 0x73, 0x20, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x20, 0x73, 0x74, 0x65, 0x70, 0x73, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x73, 0x65, 0x74, 0x5f, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x70, 0x79, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x75, 0x73, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x6a, 0x6f, 0x69, 0x6e, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x70, 0x79, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x72, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x33, 0x2c, 0x20, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x72, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x6c, 0x6f, 0x77, 0x5f, 0x70, 0x61, 0x73, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x28, 0x72, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x32, 0x31, 0x2c, 0x20, 0x33, 0x31, 0x2c, 0x20, 0x32, 0x2e, 0x35, 0x2c, 0x20, 0x35, 0x2e, 0x30, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x75, 0x70, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x28, 0x72, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x33, 0x2c, 0x20, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x72, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x49, 0x6e, 0x20, 0x63, 0x61, 0x73, 0x65, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x20, 0x69, 0x73, 0x20, 0x72, 0x75, 0x6e, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x69, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x20, 0x72, 0x66, 0x69, 0x67, 0x75, 0x69, 0x2c, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x69, 0x6e, 0x67, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x74, 0x68, 0x65, 0x20, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x20, 0x76, 0x69, 0x73, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x77, 0x69, 0x6c, 0x6c, 0x20, 0x61, 0x64, 0x64, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x20, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6c, 0x69, 0x73, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x20, 0x76, 0x69, 0x73, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x49, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x20, 0x69, 0x73, 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x72, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x20, 0x69, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x20, 0x72, 0x66, 0x69, 0x67, 0x75, 0x69, 0x2c, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x20, 0x69, 0x73, 0x20, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x64, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x76, 0x69, 0x73, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x22, 0x46, 0x69, 0x74, 0x20, 0x23, 0x22, 0x20, 0x2e, 0x2e, 0x20, 0x69, 0x2c, 0x20, 0x69, 0x20, 0x2d, 0x20, 0x31, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x20, 0x74, 0x6d, 0x70, 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x70, 0x79, 0x20, 0x2d, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x6d, 0x70, 0x3a, 0x73, 0x65, 0x74, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x74, 0x6d, 0x70, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x76, 0x69, 0x73, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x22, 0x52, 0x65, 0x73, 0x69, 0x64, 0x75, 0x61, 0x6c, 0x20, 0x23, 0x22, 0x20, 0x2e, 0x2e, 0x20, 0x69, 0x2c, 0x20, 0x69, 0x20, 0x2b, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x28, 0x28, 0x69, 0x70, 0x6f, 0x6c, 0x20, 0x2d, 0x20, 0x31, 0x29, 0x20, 0x2a, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x20, 0x2b, 0x20, 0x69, 0x2c, 0x20, 0x23, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x2a, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x20, 0x2d, 0x2d, 0x20, 0x65, 0x6e, 0x64, 0x20, 0x6f, 0x66, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x75, 0x73, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x73, 0x75, 0x6d, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x65, 0x64, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x70, 0x79, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x20, 0x2a, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x75, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x73, 0x75, 0x6d, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x2c, 0x20, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x20, 0x2a, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x20, 0x2d, 0x2d, 0x20, 0x65, 0x6e, 0x64, 0x20, 0x6f, 0x66, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x20, 0x72, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x75, 0x73, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x6a, 0x6f, 0x69, 0x6e, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x70, 0x79, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x48, 0x65, 0x6c, 0x70, 0x65, 0x72, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, 0x64, 0x20, 0x62, 0x65, 0x6c, 0x6f, 0x77, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x28, 0x61, 0x72, 0x72, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x5f, 0x2c, 0x20, 0x76, 0x20, 0x69, 0x6e, 0x20, 0x69, 0x70, 0x61, 0x69, 0x72, 0x73, 0x28, 0x61, 0x72, 0x72, 0x29, 0x20, 0x64, 0x6f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x76, 0x20, 0x3d, 0x3d, 0x20, 0x76, 0x61, 0x6c, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x28, 0x69, 0x6e, 0x70, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2c, 0x20, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x29, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x69, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x28, 0x29, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x5f, 0x74, 0x6f, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x28, 0x22, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x73, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x28, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x6a, 0x6f, 0x69, 0x6e, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x76, 0x69, 0x73, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2c, 0x20, 0x22, 0x52, 0x65, 0x73, 0x69, 0x64, 0x75, 0x61, 0x6c, 0x20, 0x23, 0x22, 0x20, 0x2e, 0x2e, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x2c, 0x20, 0x32, 0x20, 0x2a, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x28, 0x69, 0x70, 0x6f, 0x6c, 0x2c, 0x20, 0x23, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x29, 0x0a, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x20, 0x2d, 0x2d, 0x20, 0x65, 0x6e, 0x64, 0x20, 0x6f, 0x66, 0x20, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x0a, 0x0a, 0x20, 0x20, 0x69, 0x66, 0x20, 0x75, 0x73, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x76, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x5f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x65, 0x64, 0x28, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x2c, 0x20, 0x63, 0x6f, 0x70, 0x79, 0x5f, 0x6f, 0x66, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x2c, 0x20, 0x30, 0x2e, 0x32, 0x2c, 0x20, 0x30, 0x2e, 0x32, 0x29, 0x0a, 0x20, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x76, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x5f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x28, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x2c, 0x20, 0x30, 0x2e, 0x32, 0x2c, 0x20, 0x30, 0x2e, 0x32, 0x29, 0x0a, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x0a, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x65, 0x70, 0x5f, 0x72, 0x6d, 0x73, 0x28, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x2c, 0x20, 0x34, 0x2e, 0x30, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x69, 0x66, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x69, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x28, 0x29, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x68, 0x61, 0x73, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x28, 0x29, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x20, 0x77, 0x69, 0x6c, 0x6c, 0x20, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x61, 0x20, 0x66, 0x65, 0x77, 0x20, 0x73, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x20, 0x6c, 0x69, 0x6b, 0x65, 0x20, 0x66, 0x6c, 0x61, 0x67, 0x25, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x74, 0x64, 0x64, 0x65, 0x76, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2c, 0x20, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x62, 0x61, 0x73, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x77, 0x72, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x6f, 0x73, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x4d, 0x53, 0x2e, 0x20, 0x54, 0x68, 0x65, 0x73, 0x65, 0x20, 0x63, 0x61, 0x6e, 0x20, 0x62, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x20, 0x76, 0x69, 0x73, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x61, 0x6f, 0x71, 0x70, 0x6c, 0x6f, 0x74, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6f, 0x66, 0x6c, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x28, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x2c, 0x20, 0x63, 0x6f, 0x70, 0x79, 0x5f, 0x6f, 0x66, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x29, 0x0a, 0x20, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x6e, 0x61, 0x6e, 0x73, 0x28, 0x29, 0x0a, 0x65, 0x6e, 0x64, 0x0a}; unsigned int data_strategies_generic_default_lua_len = 5994; aoflagger-v3.5.1/lua/tools.h0000664000175000017500000000364314752462134014067 0ustar oleole#ifndef LUA_TOOLS_H #define LUA_TOOLS_H extern "C" { #include #include #include } #include "data.h" class Tools { public: static aoflagger_lua::Data* NewData(lua_State* L, TimeFrequencyData&& tfData, TimeFrequencyMetaDataCPtr metaData, aoflagger_lua::Data::Context& context) { void* userdatum = lua_newuserdata(L, sizeof(aoflagger_lua::Data)); aoflagger_lua::Data* data = new (userdatum) aoflagger_lua::Data(std::move(tfData), std::move(metaData), context); luaL_getmetatable(L, "AOFlaggerData"); lua_setmetatable(L, -2); return data; } static aoflagger_lua::Data* NewData(lua_State* L, TimeFrequencyData& tfData, TimeFrequencyMetaDataCPtr metaData, aoflagger_lua::Data::Context& context) { void* userdatum = lua_newuserdata(L, sizeof(aoflagger_lua::Data)); aoflagger_lua::Data* data = new (userdatum) aoflagger_lua::Data(tfData, std::move(metaData), context); luaL_getmetatable(L, "AOFlaggerData"); lua_setmetatable(L, -2); return data; } static aoflagger_lua::Data* NewData(lua_State* L, aoflagger_lua::Data&& source) { void* userdatum = lua_newuserdata(L, sizeof(aoflagger_lua::Data)); aoflagger_lua::Data* data = new (userdatum) aoflagger_lua::Data(std::move(source)); luaL_getmetatable(L, "AOFlaggerData"); lua_setmetatable(L, -2); return data; } static aoflagger_lua::Data* NewData(lua_State* L, const aoflagger_lua::Data& source) { void* userdatum = lua_newuserdata(L, sizeof(aoflagger_lua::Data)); aoflagger_lua::Data* data = new (userdatum) aoflagger_lua::Data(source); luaL_getmetatable(L, "AOFlaggerData"); lua_setmetatable(L, -2); return data; } }; #endif aoflagger-v3.5.1/lua/functionswrapper.cpp0000664000175000017500000003554215131147277016676 0ustar oleole#include "functionswrapper.h" #include "data.h" #include "../algorithms/normalizebandpass.h" #include "../util/progress/progresslistener.h" #include "functions.h" #include "scriptdata.h" #include "tools.h" #include "version.h" #include "../structures/versionstring.h" using algorithms::NormalizeBandpass; int Functions::apply_bandpass(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const std::string filename = luaL_checklstring(L, 2, nullptr); lua_pushstring(L, "AOFlagger.ScriptData"); lua_gettable(L, LUA_REGISTRYINDEX); ScriptData* scriptData = reinterpret_cast(lua_touserdata(L, -1)); try { aoflagger_lua::apply_bandpass(*data, filename, *scriptData); } catch (std::exception& e) { return luaL_error(L, e.what()); } return 0; } int Functions::collect_statistics(lua_State* L) { const aoflagger_lua::Data* dataAfter = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const aoflagger_lua::Data* dataBefore = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); lua_pushstring(L, "AOFlagger.ScriptData"); lua_gettable(L, LUA_REGISTRYINDEX); ScriptData* scriptData = reinterpret_cast(lua_touserdata(L, -1)); try { aoflagger_lua::collect_statistics(*dataAfter, *dataBefore, *scriptData); } catch (std::exception& e) { return luaL_error(L, e.what()); } return 0; } int Functions::copy_to_channel(lua_State* L) { aoflagger_lua::Data* destination = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const aoflagger_lua::Data* source = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); const long channel = luaL_checkinteger(L, 3); try { aoflagger_lua::copy_to_channel(*destination, *source, channel); } catch (std::exception& e) { return luaL_error(L, e.what()); } return 0; } int Functions::copy_to_frequency(lua_State* L) { aoflagger_lua::Data* destination = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const aoflagger_lua::Data* source = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); const double frequencyMHz = luaL_checknumber(L, 3); try { aoflagger_lua::copy_to_frequency(*destination, *source, frequencyMHz * 1e6); } catch (std::exception& e) { return luaL_error(L, e.what()); } return 0; } int Functions::downsample(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); long horizontalFactor = luaL_checkinteger(L, 2), verticalFactor = luaL_checkinteger(L, 3); if (!lua_isboolean(L, 4)) { luaL_error( L, "Parameters 4 should be of boolean type in call to downsample()"); } const bool masked = lua_toboolean(L, 4); try { if (masked) Tools::NewData(L, aoflagger_lua::downsample_masked( *data, horizontalFactor, verticalFactor)); else Tools::NewData(L, aoflagger_lua::downsample(*data, horizontalFactor, verticalFactor)); } catch (std::exception& e) { return luaL_error(L, e.what()); } return 1; } int Functions::high_pass_filter(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); long kernelWidth = luaL_checkinteger(L, 2), kernelHeight = luaL_checkinteger(L, 3); double horizontalSigmaSquared = luaL_checknumber(L, 4), verticalSigmaSquared = luaL_checknumber(L, 5); aoflagger_lua::high_pass_filter(*data, kernelWidth, kernelHeight, horizontalSigmaSquared, verticalSigmaSquared); return 0; } int Functions::low_pass_filter(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); long kernelWidth = luaL_checkinteger(L, 2), kernelHeight = luaL_checkinteger(L, 3); double horizontalSigmaSquared = luaL_checknumber(L, 4), verticalSigmaSquared = luaL_checknumber(L, 5); aoflagger_lua::low_pass_filter(*data, kernelWidth, kernelHeight, horizontalSigmaSquared, verticalSigmaSquared); return 0; } int Functions::norm(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); try { aoflagger_lua::Data result = aoflagger_lua::norm(*data); Tools::NewData(L, std::move(result)); } catch (std::exception& e) { luaL_error(L, e.what()); } return 1; } int Functions::normalize_bandpass(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); NormalizeBandpass::NormalizeSmooth(data->TFData()); return 0; } int Functions::normalize_subbands(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const long nSubbands = luaL_checkinteger(L, 2); NormalizeBandpass::NormalizeStepwise(data->TFData(), nSubbands); return 0; } int Functions::print_polarization_statistics(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); aoflagger_lua::print_polarization_statistics(*data); return 0; } int Functions::require_min_version(lua_State* L) { VersionString minVersion; try { minVersion = VersionString(luaL_checkstring(L, 1)); } catch (std::exception& e) { return luaL_error(L, e.what()); } bool met = minVersion.Major() < AOFLAGGER_VERSION_MAJOR; if (minVersion.Major() == AOFLAGGER_VERSION_MAJOR) { if (!minVersion.HasMinor() || minVersion.Minor() < AOFLAGGER_VERSION_MINOR) met = true; else if (minVersion.Minor() == AOFLAGGER_VERSION_MINOR && (!minVersion.HasSubminor() || minVersion.Subminor() <= AOFLAGGER_VERSION_SUBMINOR)) met = true; } if (!met) { const std::string err = std::string( "Requirements on AOFlagger version not met: This " "is " AOFLAGGER_VERSION_STR ", required is version >= ") + minVersion.String(); luaL_error(L, err.c_str()); } return 0; } int Functions::require_max_version(lua_State* L) { VersionString maxVersion; try { maxVersion = VersionString(luaL_checkstring(L, 1)); } catch (std::exception& e) { return luaL_error(L, e.what()); } bool met = maxVersion.Major() > AOFLAGGER_VERSION_MAJOR; if (maxVersion.Major() == AOFLAGGER_VERSION_MAJOR) { if (!maxVersion.HasMinor() || maxVersion.Minor() > AOFLAGGER_VERSION_MINOR) met = true; else if (maxVersion.Minor() == AOFLAGGER_VERSION_MINOR && (!maxVersion.HasSubminor() || maxVersion.Subminor() >= AOFLAGGER_VERSION_SUBMINOR)) met = true; } if (!met) { const std::string err = std::string( "Requirements on AOFlagger version not met: This " "is " AOFLAGGER_VERSION_STR ", required is version <= ") + maxVersion.String(); luaL_error(L, err.c_str()); } return 0; } int Functions::save_heat_map(lua_State* L) { const char* str = luaL_checklstring(L, 1, nullptr); aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); aoflagger_lua::save_heat_map(str, *data); return 0; } int Functions::scale_invariant_rank_operator(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const double level_horizontal = luaL_checknumber(L, 2); const double level_vertical = luaL_checknumber(L, 3); try { aoflagger_lua::scale_invariant_rank_operator(*data, level_horizontal, level_vertical); } catch (std::exception& e) { luaL_error(L, e.what()); } return 0; } int Functions::scale_invariant_rank_operator_masked(lua_State* L) { const bool hasPenalty = lua_gettop(L) >= 5; aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const aoflagger_lua::Data* missing = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); const double level_horizontal = luaL_checknumber(L, 3); const double level_vertical = luaL_checknumber(L, 4); const double penalty = hasPenalty ? luaL_checknumber(L, 5) : 0.1; try { aoflagger_lua::scale_invariant_rank_operator_masked( *data, *missing, level_horizontal, level_vertical, penalty); } catch (std::exception& e) { luaL_error(L, e.what()); } return 0; } int Functions::set_progress(lua_State* L) { double progress = luaL_checkinteger(L, 1), maxProgress = luaL_checkinteger(L, 2); lua_pushstring(L, "AOFlagger.ScriptData"); lua_gettable(L, LUA_REGISTRYINDEX); ScriptData* scriptData = reinterpret_cast(lua_touserdata(L, -1)); if (scriptData->Progress()) scriptData->Progress()->OnProgress(progress, maxProgress); return 0; } int Functions::set_progress_text(lua_State* L) { const std::string str = luaL_checklstring(L, 1, nullptr); lua_pushstring(L, "AOFlagger.ScriptData"); lua_gettable(L, LUA_REGISTRYINDEX); ScriptData* scriptData = reinterpret_cast(lua_touserdata(L, -1)); if (scriptData->Progress()) scriptData->Progress()->OnStartTask(str); return 0; } int Functions::sqrt(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); try { aoflagger_lua::Data result = aoflagger_lua::sqrt(*data); Tools::NewData(L, std::move(result)); } catch (std::exception& e) { luaL_error(L, e.what()); } return 1; } int Functions::sumthreshold(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); double hThresholdFactor = luaL_checknumber(L, 2), vThresholdFactor = luaL_checknumber(L, 3); if (!lua_isboolean(L, 4) || !lua_isboolean(L, 5)) { luaL_error(L, "Parameters 4 and 5 should be of boolean type in call to " "sumthreshold()"); } else { bool horizontal = lua_toboolean(L, 4), vertical = lua_toboolean(L, 5); aoflagger_lua::sumthreshold(*data, hThresholdFactor, vThresholdFactor, horizontal, vertical); } return 0; } int Functions::sumthreshold_masked(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const aoflagger_lua::Data* missing = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); double hThresholdFactor = luaL_checknumber(L, 3), vThresholdFactor = luaL_checknumber(L, 4); if (!lua_isboolean(L, 5) || !lua_isboolean(L, 6)) { luaL_error(L, "Parameters 5 and 6 should be of boolean type in call to " "sumthreshold()"); } else { bool horizontal = lua_toboolean(L, 4), vertical = lua_toboolean(L, 5); aoflagger_lua::sumthreshold_masked(*data, *missing, hThresholdFactor, vThresholdFactor, horizontal, vertical); } return 0; } int Functions::threshold_channel_rms(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const double threshold = luaL_checknumber(L, 2); if (!lua_isboolean(L, 3)) { luaL_error(L, "3rd parameter should be of boolean type in call to " "threshold_channel_rms()"); } else { const bool thresholdLowValues = lua_toboolean(L, 3); aoflagger_lua::threshold_channel_rms(*data, threshold, thresholdLowValues); } return 0; } int Functions::threshold_timestep_rms(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const double threshold = luaL_checknumber(L, 2); aoflagger_lua::threshold_timestep_rms(*data, threshold); return 0; } int Functions::trim_channels(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const long start_channel = luaL_checkinteger(L, 2); const long end_channel = luaL_checkinteger(L, 3); try { aoflagger_lua::Data trimmed_data = aoflagger_lua::trim_channels(*data, start_channel, end_channel); Tools::NewData(L, std::move(trimmed_data)); } catch (std::exception& e) { luaL_error(L, e.what()); } return 1; } int Functions::trim_frequencies(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const double start_frequency = luaL_checknumber(L, 2); const double end_frequency = luaL_checknumber(L, 3); try { aoflagger_lua::Data trimmed_data = aoflagger_lua::trim_frequencies( *data, start_frequency * 1e6, end_frequency * 1e6); Tools::NewData(L, std::move(trimmed_data)); } catch (std::exception& e) { luaL_error(L, e.what()); } return 1; } int Functions::upsample_image(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); aoflagger_lua::Data* destination = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); long horizontalFactor = luaL_checkinteger(L, 3), verticalFactor = luaL_checkinteger(L, 4); try { aoflagger_lua::upsample_image(*data, *destination, horizontalFactor, verticalFactor); } catch (std::exception& e) { luaL_error(L, e.what()); } return 0; } int Functions::upsample_mask(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); aoflagger_lua::Data* destination = reinterpret_cast( luaL_checkudata(L, 2, "AOFlaggerData")); long horizontalFactor = luaL_checkinteger(L, 3), verticalFactor = luaL_checkinteger(L, 4); try { aoflagger_lua::upsample_mask(*data, *destination, horizontalFactor, verticalFactor); } catch (std::exception& e) { luaL_error(L, e.what()); } return 0; } int Functions::visualize(lua_State* L) { aoflagger_lua::Data* data = reinterpret_cast( luaL_checkudata(L, 1, "AOFlaggerData")); const std::string label = luaL_checklstring(L, 2, nullptr); const long sortingIndex = luaL_checkinteger(L, 3); lua_pushstring(L, "AOFlagger.ScriptData"); lua_gettable(L, LUA_REGISTRYINDEX); ScriptData* scriptData = reinterpret_cast(lua_touserdata(L, -1)); scriptData->AddVisualization(data->TFData(), label, sortingIndex); return 0; } aoflagger-v3.5.1/lua/telescopefile.cpp0000664000175000017500000001071314752462134016101 0ustar oleole#include "telescopefile.h" #include #include #include #include "../imagesets/bhfitsimageset.h" #include "../imagesets/filterbankset.h" #include "../imagesets/fitsimageset.h" #include "../imagesets/imageset.h" #include "../imagesets/msimageset.h" #include "../structures/msmetadata.h" #include std::string TelescopeFile::TelescopeName( TelescopeFile::TelescopeId telescopeId) { switch (telescopeId) { case GENERIC_TELESCOPE: return "Generic"; case AARTFAAC_TELESCOPE: return "Aartfaac"; case APERTIF_TELESCOPE: return "APERTIF"; case ARECIBO_TELESCOPE: return "Arecibo"; case ATCA_TELESCOPE: return "ATCA"; case BIGHORNS_TELESCOPE: return "Bighorns"; case JVLA_TELESCOPE: return "JVLA"; case LOFAR_TELESCOPE: return "LOFAR"; case MWA_TELESCOPE: return "MWA"; case NENUFAR_TELESCOPE: return "NenuFAR"; case PARKES_TELESCOPE: return "Parkes"; case WSRT_TELESCOPE: return "WSRT"; } return ""; } std::string TelescopeFile::TelescopeDescription( TelescopeFile::TelescopeId telescopeId) { switch (telescopeId) { default: case GENERIC_TELESCOPE: return "Generic"; case AARTFAAC_TELESCOPE: return "AARTFAAC"; case APERTIF_TELESCOPE: return "WSRT with APERTIF multi-beaming system"; case ARECIBO_TELESCOPE: return "Arecibo (305 m single dish, Puerto Rico)"; case BIGHORNS_TELESCOPE: return "Bighorns (low-frequency wide-band EoR instrument, Curtin uni, " "Australia)"; case JVLA_TELESCOPE: return "JVLA (Jansky Very Large Array, New Mexico)"; case LOFAR_TELESCOPE: return "LOFAR (Low-Frequency Array, Europe)"; case MWA_TELESCOPE: return "MWA (Murchison Widefield Array, Australia)"; case NENUFAR_TELESCOPE: return "NenuFAR (New Extension in Nançay upgrading LOFAR, France)"; case PARKES_TELESCOPE: return "Parkes (single dish, Australia)"; case WSRT_TELESCOPE: return "WSRT (Westerbork Synthesis Radio Telescope, Netherlands)"; } } TelescopeFile::TelescopeId TelescopeFile::TelescopeIdFromName( const std::string& name) { const std::string nameUpper = boost::algorithm::to_upper_copy(name); if (nameUpper == "AARTFAAC") return AARTFAAC_TELESCOPE; else if (nameUpper == "APERTIF") return APERTIF_TELESCOPE; else if (nameUpper == "ARECIBO" || nameUpper == "ARECIBO 305M") return ARECIBO_TELESCOPE; else if (nameUpper == "ATCA") return ATCA_TELESCOPE; else if (nameUpper == "BIGHORNS") return BIGHORNS_TELESCOPE; else if (nameUpper == "EVLA" || nameUpper == "JVLA") return JVLA_TELESCOPE; else if (nameUpper == "LOFAR") return LOFAR_TELESCOPE; else if (nameUpper == "MWA") return MWA_TELESCOPE; else if (nameUpper == "NENUFAR") return NENUFAR_TELESCOPE; else if (nameUpper == "PARKES" || nameUpper == "PKS" || nameUpper == "ATPKSMB") return PARKES_TELESCOPE; else if (nameUpper == "WSRT") return WSRT_TELESCOPE; else return GENERIC_TELESCOPE; } std::string TelescopeFile::FindStrategy(enum TelescopeId telescopeId, const std::string& scenario) { std::string filename = boost::to_lower_copy(TelescopeName(telescopeId)); if (scenario.empty()) filename = filename + "-default.lua"; else filename = filename + "-" + boost::to_lower_copy(scenario) + ".lua"; std::filesystem::path search; search = std::filesystem::path(AOFLAGGER_INSTALL_PATH) / "share/aoflagger/strategies" / filename; if (std::filesystem::exists(search)) return search.string(); // Try using the directory of the AOFlagger library. // When bundled as a python binary wheel, the strategies are there. Dl_info dl_info; if (dladdr(reinterpret_cast(&TelescopeFile::FindStrategy), &dl_info)) { std::filesystem::path aoflagger_library_path(dl_info.dli_fname); search = aoflagger_library_path.remove_filename() / "aoflagger/strategies" / filename; if (std::filesystem::exists(search)) return search.string(); } search = std::filesystem::path("/usr/share/aoflagger/strategies") / filename; if (std::filesystem::exists(search)) return search.string(); search = std::filesystem::path("/usr/local/share/aoflagger/strategies") / filename; if (std::filesystem::exists(search)) return search.string(); return std::string(); } aoflagger-v3.5.1/lua/data.h0000664000175000017500000001066014752462134013635 0ustar oleole#ifndef LUA_DATA_H #define LUA_DATA_H #include "../structures/timefrequencydata.h" #include "../structures/timefrequencymetadata.h" #include #include #include namespace aoflagger_lua { class Data { public: struct Context { std::vector list; Context() = default; Context(const Context&) = delete; Context& operator=(const Context&) = delete; Context(Context&&) = default; Context& operator=(Context&&) = default; }; explicit Data(Context& context) : _context(&context), _persistent(false) { context.list.emplace_back(this); } Data(const Data& source) : _tfData(source._tfData), _metaData(source._metaData), _context(source._context), _persistent(source._persistent) { _context->list.emplace_back(this); } Data(Data&& source) : _tfData(std::move(source._tfData)), _metaData(std::move(source._metaData)), _context(source._context), _persistent(source._persistent) { _context->list.emplace_back(this); } Data(const TimeFrequencyData& tfData, TimeFrequencyMetaDataCPtr metaData, Context& context) : _tfData(tfData), _metaData(metaData), _context(&context), _persistent(false) { context.list.emplace_back(this); } Data(TimeFrequencyData&& tfData, TimeFrequencyMetaDataCPtr metaData, Context& context) noexcept : _tfData(std::move(tfData)), _metaData(metaData), _context(&context), _persistent(false) { context.list.emplace_back(this); } ~Data() noexcept { // If this object is persistent, it might exist after the context was // deleted In that case, we don't update context. if (!_persistent) { auto iter = std::find(_context->list.begin(), _context->list.end(), this); // The list might be cleared already (at the end of the strategy), so make // sure the data was found. if (iter != _context->list.end()) _context->list.erase(iter); } } Data& operator=(Data&& rhs) { _tfData = std::move(rhs._tfData); _metaData = std::move(rhs._metaData); _context = rhs._context; _persistent = rhs._persistent; return *this; } Data operator-(const Data& other) const { return Data(TimeFrequencyData::MakeFromDiff(_tfData, other.TFData()), _metaData, *_context); } void clear() { _tfData = TimeFrequencyData(); _metaData.reset(); } void clear_mask() noexcept { _tfData.SetNoMask(); } Data convert_to_polarization(aocommon::PolarizationEnum polarization) const { return Data(_tfData.Make(polarization), _metaData, *_context); } Data convert_to_complex(enum TimeFrequencyData::ComplexRepresentation complexRepresentation) const { return Data(_tfData.Make(complexRepresentation), _metaData, *_context); } Data copy() const { return Data(_tfData, _metaData, *_context); } void join_mask(const Data& other) { _tfData.JoinMask(other._tfData); } Data make_complex() const { return Data(_tfData.MakeFromComplexCombination(_tfData, _tfData), _metaData, *_context); } void set_visibilities(const Data& image_data) { const TimeFrequencyData& source = image_data._tfData; const size_t imageCount = source.ImageCount(); if (imageCount != _tfData.ImageCount()) { std::ostringstream s; s << "set_image() was executed with incompatible polarizations: input " "had " << imageCount << ", destination had " << _tfData.ImageCount(); throw std::runtime_error(s.str()); } for (size_t i = 0; i != imageCount; ++i) _tfData.SetImage(i, source.GetImage(i)); } void set_polarization_data(aocommon::PolarizationEnum polarization, const Data& data) { size_t polIndex = _tfData.GetPolarizationIndex(polarization); _tfData.SetPolarizationData(polIndex, data._tfData); } TimeFrequencyData& TFData() noexcept { return _tfData; } const TimeFrequencyData& TFData() const noexcept { return _tfData; } const TimeFrequencyMetaDataCPtr& MetaData() const noexcept { return _metaData; } Context& GetContext() const { return *_context; } bool is_persistent() const { return _persistent; } void set_persistent(bool persistent) { _persistent = persistent; } private: TimeFrequencyData _tfData; TimeFrequencyMetaDataCPtr _metaData; Context* _context; bool _persistent; }; } // namespace aoflagger_lua #endif aoflagger-v3.5.1/lua/functionswrapper.h0000664000175000017500000000265315131147277016340 0ustar oleole#ifndef LUA_FUNCTIONS_WRAPPER_H #define LUA_FUNCTIONS_WRAPPER_H extern "C" { #include #include #include } class Functions { public: static int apply_bandpass(lua_State* L); static int collect_statistics(lua_State* L); static int copy_to_channel(lua_State* L); static int copy_to_frequency(lua_State* L); static int downsample(lua_State* L); static int high_pass_filter(lua_State* L); static int low_pass_filter(lua_State* L); static int norm(lua_State* L); static int normalize_bandpass(lua_State* L); static int normalize_subbands(lua_State* L); static int print_polarization_statistics(lua_State* L); static int require_min_version(lua_State* L); static int require_max_version(lua_State* L); static int save_heat_map(lua_State* L); static int scale_invariant_rank_operator(lua_State* L); static int scale_invariant_rank_operator_masked(lua_State* L); static int set_progress(lua_State* L); static int set_progress_text(lua_State* L); static int sqrt(lua_State* L); static int sumthreshold(lua_State* L); static int sumthreshold_masked(lua_State* L); static int threshold_channel_rms(lua_State* L); static int threshold_timestep_rms(lua_State* L); static int trim_channels(lua_State* L); static int trim_frequencies(lua_State* L); static int upsample_image(lua_State* L); static int upsample_mask(lua_State* L); static int visualize(lua_State* L); }; #endif aoflagger-v3.5.1/lua/telescopefile.h0000664000175000017500000000437514752462134015555 0ustar oleole#ifndef DEFAULTSTRATEGYSET_H #define DEFAULTSTRATEGYSET_H #include #include #include #include "../interface/aoflagger.h" class TelescopeFile { public: /** * The contents of this enum needs to be equal to aoflagger::StrategyId * defined in interfaces/aoflagger.h */ enum TelescopeId { GENERIC_TELESCOPE = (int)aoflagger::TelescopeId::GENERIC_TELESCOPE, AARTFAAC_TELESCOPE = (int)aoflagger::TelescopeId::AARTFAAC_TELESCOPE, APERTIF_TELESCOPE = (int)aoflagger::TelescopeId::APERTIF_TELESCOPE, ARECIBO_TELESCOPE = (int)aoflagger::TelescopeId::ARECIBO_TELESCOPE, ATCA_TELESCOPE = (int)aoflagger::TelescopeId::ATCA_TELESCOPE, BIGHORNS_TELESCOPE = (int)aoflagger::TelescopeId::BIGHORNS_TELESCOPE, JVLA_TELESCOPE = (int)aoflagger::TelescopeId::JVLA_TELESCOPE, LOFAR_TELESCOPE = (int)aoflagger::TelescopeId::LOFAR_TELESCOPE, MWA_TELESCOPE = (int)aoflagger::TelescopeId::MWA_TELESCOPE, NENUFAR_TELESCOPE = (int)aoflagger::TelescopeId::NENUFAR_TELESCOPE, PARKES_TELESCOPE = (int)aoflagger::TelescopeId::PARKES_TELESCOPE, WSRT_TELESCOPE = (int)aoflagger::TelescopeId::WSRT_TELESCOPE }; static std::vector List() { return std::vector{ GENERIC_TELESCOPE, AARTFAAC_TELESCOPE, APERTIF_TELESCOPE, ARECIBO_TELESCOPE, ATCA_TELESCOPE, BIGHORNS_TELESCOPE, JVLA_TELESCOPE, LOFAR_TELESCOPE, MWA_TELESCOPE, NENUFAR_TELESCOPE, PARKES_TELESCOPE, WSRT_TELESCOPE}; } /** * @brief Searches a strategy for a given telescope. * * @param telescopeId One of the telescopes, if known, otherwise * GENERIC_TELESCOPE. * @param scenario Used as 'suffix' to the name of the telescope. This allows * multiple versions for the same telescope. * @returns Path to the strategy, or empty string if not found. */ static std::string FindStrategy(enum TelescopeId telescopeId, const std::string& scenario = ""); static std::string TelescopeName(TelescopeFile::TelescopeId telescopeId); static std::string TelescopeDescription( TelescopeFile::TelescopeId telescopeId); static TelescopeFile::TelescopeId TelescopeIdFromName( const std::string& name); }; #endif aoflagger-v3.5.1/msio/0000775000175000017500000000000015146315735012740 5ustar oleoleaoflagger-v3.5.1/msio/indirectbaselinereader.h0000664000175000017500000000000015136346710017562 0ustar oleoleaoflagger-v3.5.1/msio/baselinematrixloader.h0000664000175000017500000000214214752462134017304 0ustar oleole#ifndef BASELINEMATRIXLOADER_H #define BASELINEMATRIXLOADER_H #include #include #include "../structures/timefrequencydata.h" #include "../structures/msmetadata.h" /** * Loader for antenna x antenna matrices, useful for e.g. spatial analyses such * as spatial filtering. */ class BaselineMatrixLoader { public: explicit BaselineMatrixLoader(MSMetaData& measurementSet); TimeFrequencyData Load(size_t timeIndex) { return LoadSummed(timeIndex); } void LoadPerChannel(size_t timeIndex, std::vector& data); size_t TimeIndexCount() const { return _timeIndexCount; } class SpatialMatrixMetaData& MetaData() const { return *_metaData; } size_t FrequencyCount() const { return _frequencyCount; } private: TimeFrequencyData LoadSummed(size_t timeIndex); std::unique_ptr _sortedTable; std::unique_ptr _tableIter; size_t _currentIterIndex; MSMetaData _msMetaData; size_t _timeIndexCount; std::unique_ptr _metaData; size_t _frequencyCount; }; #endif aoflagger-v3.5.1/msio/rawdescfile.h0000664000175000017500000000472314752462134015405 0ustar oleole#ifndef RAWDESCFILE_H #define RAWDESCFILE_H #include #include #include #include "../util/logger.h" class RawDescFile { public: explicit RawDescFile(const std::string& filename) : _filename(filename) { readFile(); } size_t GetCount() const { return _sets.size(); } std::string GetSet(size_t index) const { return _sets[index]; } const std::string& Filename() const { return _filename; } unsigned BeamCount() const { return _beamCount; } unsigned SubbandCount() const { return _subbandCount; } unsigned ChannelsPerSubbandCount() const { return _channelsPerSubbandCount; } unsigned TimestepsPerBlockCount() const { return _timestepsPerBlockCount; } unsigned BlockHeaderSize() const { return _blockHeaderSize; } unsigned BlockFooterSize() const { return _blockFooterSize; } unsigned SelectedBeam() const { return _selectedBeam; } double TimeResolution() const { return _timeRes; } double DisplayedTimeDuration() const { return _displayedTimeDuration; } double FrequencyResolution() const { return _freqRes; } double FrequencyStart() const { return _freqStart; } private: const std::string _filename; std::vector _sets; unsigned _beamCount; unsigned _subbandCount; unsigned _channelsPerSubbandCount; unsigned _timestepsPerBlockCount; unsigned _blockHeaderSize; unsigned _blockFooterSize; unsigned _selectedBeam; double _timeRes; double _displayedTimeDuration; double _freqRes; double _freqStart; void readFile() { std::ifstream file(_filename.c_str()); std::string l; std::getline(file, l); _beamCount = (unsigned)atol(l.c_str()); std::getline(file, l); _subbandCount = (unsigned)atol(l.c_str()); std::getline(file, l); _channelsPerSubbandCount = (unsigned)atol(l.c_str()); std::getline(file, l); _timestepsPerBlockCount = (unsigned)atol(l.c_str()); std::getline(file, l); _blockHeaderSize = (unsigned)atol(l.c_str()); std::getline(file, l); _blockFooterSize = (unsigned)atol(l.c_str()); std::getline(file, l); _selectedBeam = (unsigned)atol(l.c_str()); std::getline(file, l); _timeRes = atof(l.c_str()); std::getline(file, l); _displayedTimeDuration = atof(l.c_str()); std::getline(file, l); _freqStart = atof(l.c_str()); std::getline(file, l); _freqRes = atof(l.c_str()); while (file.good()) { std::getline(file, l); if (l != "") _sets.push_back(l); } } }; #endif aoflagger-v3.5.1/msio/fitsfile.cpp0000664000175000017500000004327314752462134015260 0ustar oleole#include "fitsfile.h" #include #include #include #include #include #include "../util/logger.h" FitsFile::FitsFile(const std::string& filename) : _filename(filename), _fptr(nullptr), _isOpen(false) {} FitsFile::~FitsFile() { if (_isOpen) Close(); } void FitsFile::CheckStatus(int status) const { if (status) { /* fits_get_errstatus returns at most 30 characters */ char err_text[31]; fits_get_errstatus(status, err_text); char err_msg[81]; std::stringstream errMsg; errMsg << "CFITSIO reported error when performing IO on file '" << _filename << "':" << err_text << " ("; while (fits_read_errmsg(err_msg)) errMsg << err_msg; errMsg << ')'; throw FitsIOException(errMsg.str()); } } void FitsFile::CheckOpen() const { if (!_isOpen) throw FitsIOException("No open file, call Open() first"); } void FitsFile::Open(FitsFile::FileMode mode) { if (_isOpen) { throw FitsIOException("File was opened twice"); } else { int status = 0; int modeInt = 0; switch (mode) { case ReadOnlyMode: modeInt = READONLY; break; case ReadWriteMode: modeInt = READWRITE; break; default: throw FitsIOException("Incorrect mode specified"); break; } fits_open_diskfile(&_fptr, _filename.c_str(), modeInt, &status); CheckStatus(status); _isOpen = true; } } void FitsFile::Create() { if (_isOpen) { throw FitsIOException("File was opened twice"); } else { int status = 0; fits_create_file(&_fptr, (std::string("!") + _filename).c_str(), &status); CheckStatus(status); _isOpen = true; } } void FitsFile::Close() { if (_isOpen) { int status = 0; fits_close_file(_fptr, &status); CheckStatus(status); _isOpen = false; _fptr = nullptr; } else { throw FitsIOException("Non-opened file was closed"); } } int FitsFile::GetHDUCount() { CheckOpen(); int hdunum = 0, status = 0; fits_get_num_hdus(_fptr, &hdunum, &status); CheckStatus(status); return hdunum; } int FitsFile::GetCurrentHDU() { CheckOpen(); int hdunum = 0; fits_get_hdu_num(_fptr, &hdunum); return hdunum; } void FitsFile::MoveToHDU(int hduNumber) { CheckOpen(); int status = 0; fits_movabs_hdu(_fptr, hduNumber, nullptr, &status); CheckStatus(status); } enum FitsFile::HDUType FitsFile::GetCurrentHDUType() { CheckOpen(); int hdutypeInt = 0, status = 0; fits_get_hdu_type(_fptr, &hdutypeInt, &status); CheckStatus(status); enum HDUType hduType; switch (hdutypeInt) { case IMAGE_HDU: hduType = ImageHDUType; break; case ASCII_TBL: hduType = ASCIITableHDUType; break; case BINARY_TBL: hduType = BinaryTableHDUType; break; default: throw FitsIOException("Unknown HDUType returned"); } return hduType; } enum FitsFile::ImageType FitsFile::GetCurrentImageType() { CheckOpen(); int bitPixInt = 0, status = 0; fits_get_img_type(_fptr, &bitPixInt, &status); CheckStatus(status); enum ImageType imageType; switch (bitPixInt) { case BYTE_IMG: imageType = Int8ImageType; break; case SHORT_IMG: imageType = Int16ImageType; break; case LONG_IMG: imageType = Int32ImageType; break; case FLOAT_IMG: imageType = Float32ImageType; break; case DOUBLE_IMG: imageType = Double64ImageType; break; default: throw FitsIOException("Unknown image type returned"); } return imageType; } int FitsFile::GetCurrentImageDimensionCount() { CheckOpen(); int status = 0, naxis = 0; fits_get_img_dim(_fptr, &naxis, &status); CheckStatus(status); return naxis; } long FitsFile::GetCurrentImageSize(int dimension) { CheckOpen(); if (dimension > GetCurrentImageDimensionCount()) throw FitsIOException("Parameter outside range"); int status = 0; long* sizes = new long[dimension]; fits_get_img_size(_fptr, dimension, sizes, &status); const long size = sizes[dimension - 1]; delete[] sizes; CheckStatus(status); return size; } void FitsFile::ReadCurrentImageData(long startPos, num_t* buffer, long bufferSize, long double nullValue) { CheckOpen(); int status = 0, dimensions = GetCurrentImageDimensionCount(); long* firstpixel = new long[dimensions]; for (int i = 0; i < dimensions; i++) { firstpixel[i] = 1 + startPos % GetCurrentImageSize(i + 1); startPos = startPos / GetCurrentImageSize(i + 1); } double* dblbuffer = new double[bufferSize]; double dblNullValue = nullValue; int anynul = 0; fits_read_pix(_fptr, TDOUBLE, firstpixel, bufferSize, &dblNullValue, dblbuffer, &anynul, &status); for (int i = 0; i < bufferSize; i++) buffer[i] = dblbuffer[i]; delete[] dblbuffer; delete[] firstpixel; CheckStatus(status); } void FitsFile::AppendImageHUD(enum FitsFile::ImageType imageType, long width, long height) { int status = 0; int bitPixInt; switch (imageType) { case Int8ImageType: bitPixInt = BYTE_IMG; break; case Int16ImageType: bitPixInt = SHORT_IMG; break; case Int32ImageType: bitPixInt = LONG_IMG; break; case Float32ImageType: bitPixInt = FLOAT_IMG; break; case Double64ImageType: bitPixInt = DOUBLE_IMG; break; default: throw FitsIOException("?"); } long* naxes = new long[2]; naxes[0] = width; naxes[1] = height; fits_create_img(_fptr, bitPixInt, 2, naxes, &status); delete[] naxes; CheckStatus(status); } void FitsFile::WriteImage(long startPos, double* buffer, long bufferSize, double nullValue) { CheckOpen(); int status = 0, dimensions = GetCurrentImageDimensionCount(); long* firstpixel = new long[dimensions]; for (int i = 0; i < dimensions; i++) { firstpixel[i] = 1 + startPos % GetCurrentImageSize(i + 1); startPos = startPos / GetCurrentImageSize(i + 1); } fits_write_pixnull(_fptr, TDOUBLE, firstpixel, bufferSize, buffer, &nullValue, &status); delete[] firstpixel; CheckStatus(status); } void FitsFile::WriteImage(long startPos, float* buffer, long bufferSize, float nullValue) { CheckOpen(); int status = 0, dimensions = GetCurrentImageDimensionCount(); long* firstpixel = new long[dimensions]; for (int i = 0; i < dimensions; i++) { firstpixel[i] = 1 + startPos % GetCurrentImageSize(i + 1); startPos = startPos / GetCurrentImageSize(i + 1); } fits_write_pixnull(_fptr, TFLOAT, firstpixel, bufferSize, buffer, &nullValue, &status); delete[] firstpixel; CheckStatus(status); } int FitsFile::GetRowCount() { CheckOpen(); long rowCount; int status = 0; fits_get_num_rows(_fptr, &rowCount, &status); CheckStatus(status); return rowCount; } int FitsFile::GetKeywordCount() { int status = 0, keysexist; fits_get_hdrspace(_fptr, &keysexist, nullptr, &status); CheckStatus(status); return keysexist; } std::string FitsFile::GetKeyword(int keywordNumber) { char keyName[FLEN_KEYWORD], keyValue[FLEN_VALUE]; int status = 0; fits_read_keyn(_fptr, keywordNumber, keyName, keyValue, nullptr, &status); CheckStatus(status); return std::string(keyName); } std::string FitsFile::GetKeywordValue(int keywordNumber) { char keyName[FLEN_KEYWORD], keyValue[FLEN_VALUE]; int status = 0; fits_read_keyn(_fptr, keywordNumber, keyName, keyValue, nullptr, &status); CheckStatus(status); std::string val(keyValue); if (val.length() >= 2 && *val.begin() == '\'' && *val.rbegin() == '\'') { val = val.substr(1, val.length() - 2); boost::trim(val); } return val; } bool FitsFile::GetKeywordValue(const std::string& keywordName, std::string& value) { char keyValue[FLEN_VALUE]; int status = 0; fits_read_keyword(_fptr, const_cast(keywordName.c_str()), keyValue, nullptr, &status); if (status == 0) { value = std::string(keyValue); if (value.length() >= 2 && *value.begin() == '\'' && *value.rbegin() == '\'') { value = value.substr(1, value.length() - 2); boost::trim(value); } return true; } else { return false; } } std::string FitsFile::GetKeywordValue(const std::string& keywordName) { char keyValue[FLEN_VALUE]; int status = 0; fits_read_keyword(_fptr, const_cast(keywordName.c_str()), keyValue, nullptr, &status); CheckStatus(status); std::string val(keyValue); if (val.length() >= 2 && *val.begin() == '\'' && *val.rbegin() == '\'') { val = val.substr(1, val.length() - 2); boost::trim(val); } return val; } std::string FitsFile::GetKeywordComment(int keywordNumber) { char keyName[FLEN_KEYWORD], keyValue[FLEN_VALUE], keyComment[FLEN_COMMENT]; int status = 0; fits_read_keyn(_fptr, keywordNumber, keyName, keyValue, keyComment, &status); CheckStatus(status); return std::string(keyComment); } int FitsFile::GetColumnCount() { CheckOpen(); int rowCount, status = 0; fits_get_num_cols(_fptr, &rowCount, &status); CheckStatus(status); return rowCount; } int FitsFile::GetColumnType(int colNumber) { CheckOpen(); int typecode, status = 0; long repeat, width; fits_get_coltype(_fptr, colNumber, &typecode, &repeat, &width, &status); CheckStatus(status); return typecode; } int FitsFile::GetIntKeywordValue(int keywordNumber) { return atoi(GetKeyword(keywordNumber).c_str()); } int FitsFile::GetIntKeywordValue(const std::string& keywordName) { return atoi(GetKeywordValue(keywordName).c_str()); } double FitsFile::GetDoubleKeywordValue(int keywordNumber) { return atof(GetKeyword(keywordNumber).c_str()); } double FitsFile::GetDoubleKeywordValue(const std::string& keywordName) { return atof(GetKeywordValue(keywordName).c_str()); } bool FitsFile::HasGroups() { try { return GetKeywordValue("GROUPS") == "T"; } catch (FitsIOException& e) { return false; } } int FitsFile::GetGroupCount() { return GetIntKeywordValue("GCOUNT"); } int FitsFile::GetParameterCount() { return GetIntKeywordValue("PCOUNT"); } long FitsFile::GetImageSize() { long size = 1; for (int i = 2; i <= GetCurrentImageDimensionCount(); ++i) { size *= GetCurrentImageSize(i); } return size; } long FitsFile::GetGroupSize() { if (!HasGroups()) throw FitsIOException("HDU has no groups"); long size = 1; for (int i = 2; i <= GetCurrentImageDimensionCount(); ++i) { size *= GetCurrentImageSize(i); } size += GetParameterCount(); return size; } void FitsFile::ReadGroupParameters(long groupIndex, long double* parametersData) { int status = 0; const long pSize = GetParameterCount(); double* parameters = new double[pSize]; fits_read_grppar_dbl(_fptr, groupIndex + 1, 1, pSize, parameters, &status); CheckStatus(status); for (long i = 0; i < pSize; ++i) parametersData[i] = parameters[i]; delete[] parameters; } void FitsFile::ReadGroup(long groupIndex, long double* groupData) { int status = 0; const long size = GetImageSize(); const long pSize = GetParameterCount(); double* parameters = new double[pSize]; const double nulValue = std::numeric_limits::quiet_NaN(); int anynul = 0; fits_read_grppar_dbl(_fptr, groupIndex + 1, 1, pSize, parameters, &status); CheckStatus(status); for (long i = 0; i < pSize; ++i) groupData[i] = parameters[i]; delete[] parameters; double* data = new double[size]; fits_read_img_dbl(_fptr, groupIndex + 1, 1, size, nulValue, data, &anynul, &status); CheckStatus(status); for (long i = 0; i < size; ++i) groupData[pSize + i] = data[i]; delete[] data; if (anynul != 0) Logger::Warn << "There were nulls in the group\n"; } void FitsFile::ReadGroupData(long groupIndex, long double* groupData) { int status = 0; const long size = GetImageSize(); const double nulValue = std::numeric_limits::quiet_NaN(); int anynul = 0; double* data = new double[size]; fits_read_img_dbl(_fptr, groupIndex + 1, 1, size, nulValue, data, &anynul, &status); CheckStatus(status); for (long i = 0; i < size; ++i) groupData[i] = data[i]; delete[] data; if (anynul != 0) Logger::Warn << "There were nulls in the group data\n"; } int FitsFile::GetGroupParameterIndex(const std::string& parameterName) { if (!HasGroups()) throw FitsIOException("HDU has no groups"); const int parameterCount = GetParameterCount(); for (int i = 1; i <= parameterCount; ++i) { std::stringstream s; s << "PTYPE" << i; if (GetKeywordValue(s.str()) == parameterName) return i - 1; } throw FitsIOException(std::string("Can not find parameter with name ") + parameterName); } int FitsFile::GetGroupParameterIndex(const std::string& parameterName, int number) { if (!HasGroups()) throw FitsIOException("HDU has no groups"); const int parameterCount = GetParameterCount(); for (int i = 1; i <= parameterCount; ++i) { std::stringstream s; s << "PTYPE" << i; if (GetKeywordValue(s.str()) == parameterName) { --number; if (number == 0) return i - 1; } } throw FitsIOException(std::string("Can not find parameter with name ") + parameterName); } bool FitsFile::HasGroupParameter(const std::string& parameterName) { if (!HasGroups()) return false; const int parameterCount = GetParameterCount(); for (int i = 1; i <= parameterCount; ++i) { std::stringstream s; s << "PTYPE" << i; if (GetKeywordValue(s.str()) == parameterName) return true; } return false; } bool FitsFile::HasGroupParameter(const std::string& parameterName, int number) { if (!HasGroups()) return false; const int parameterCount = GetParameterCount(); for (int i = 1; i <= parameterCount; ++i) { std::stringstream s; s << "PTYPE" << i; if (GetKeywordValue(s.str()) == parameterName) { --number; if (number == 0) return true; } } return false; } bool FitsFile::HasTableColumn(const std::string& columnName, int& columnIndex) { const int colCount = GetColumnCount(); for (int i = 1; i <= colCount; ++i) { std::stringstream s; s << "TTYPE" << i; if (GetKeywordValue(s.str()) == columnName) { columnIndex = i; return true; } } return false; } int FitsFile::GetTableColumnIndex(const std::string& columnName) { const int colCount = GetColumnCount(); for (int i = 1; i <= colCount; ++i) { std::stringstream s; s << "TTYPE" << i; if (GetKeywordValue(s.str()) == columnName) return i; } throw FitsIOException(std::string("Can not find column with name ") + columnName); } int FitsFile::GetTableColumnArraySize(int columnIndex) { CheckOpen(); int typecode = 0, status = 0; long repeat = 0, width = 0; fits_get_coltype(_fptr, columnIndex, &typecode, &repeat, &width, &status); CheckStatus(status); return repeat; } std::vector FitsFile::GetColumnDimensions(int columnIndex) { CheckOpen(); int naxis = 0, status = 0; constexpr int maxdim = 10; std::vector axes(maxdim, 0); fits_read_tdim(_fptr, columnIndex, maxdim, &naxis, axes.data(), &status); CheckStatus(status); axes.resize(naxis); return axes; } long FitsFile::GetColumnDimensionSize(int columnIndex, int dimension) { CheckOpen(); int naxis = 0, status = 0, maxdim = 10; long naxes[10]; for (size_t i = 0; i != 10; ++i) naxes[i] = 0; fits_read_tdim(_fptr, columnIndex, maxdim, &naxis, naxes, &status); CheckStatus(status); if (dimension >= naxis) throw FitsIOException( "Requested dimension index not available in fits file"); return naxes[dimension]; } std::string FitsFile::GetTableDimensionName(int index) { CheckOpen(); std::ostringstream name; name << "CTYPE" << (index + 1); int status = 0; char valueStr[256], commentStr[256]; fits_read_key(_fptr, TSTRING, name.str().c_str(), valueStr, commentStr, &status); std::string val; if (!status) { val = valueStr; } return val; } void FitsFile::ReadTableCell(int row, int col, double* output, size_t size) { int status = 0; double nulValue = std::numeric_limits::quiet_NaN(); int anynul = 0; fits_read_col(_fptr, TDOUBLE, col, row, 1, size, &nulValue, output, &anynul, &status); } void FitsFile::ReadTableCell(int row, int col, long double* output, size_t size) { std::vector data(size); int status = 0; double nulValue = std::numeric_limits::quiet_NaN(); int anynul = 0; fits_read_col(_fptr, TDOUBLE, col, row, 1, size, &nulValue, data.data(), &anynul, &status); for (size_t i = 0; i < size; ++i) output[i] = data[i]; } void FitsFile::ReadTableCell(int row, int col, bool* output, size_t size) { std::vector data(size); int status = 0; char nulValue = 0; int anynul = 0; fits_read_col(_fptr, TBIT, col, row, 1, size, &nulValue, data.data(), &anynul, &status); for (size_t i = 0; i < size; ++i) output[i] = data[i] != 0; } void FitsFile::ReadTableCell(int row, int col, char* output) { int status = 0; double nulValue = std::numeric_limits::quiet_NaN(); int anynul = 0; fits_read_col(_fptr, TSTRING, col, row, 1, 1, &nulValue, &output, &anynul, &status); } void FitsFile::WriteTableCell(int row, int col, double* data, size_t size) { int status = 0; fits_write_col(_fptr, TDOUBLE, col, row, 1, size, data, &status); CheckStatus(status); } void FitsFile::WriteTableCell(int row, int col, const bool* data, size_t size) { std::vector dataChar(size); int status = 0; for (size_t i = 0; i < size; ++i) { dataChar[i] = data[i] ? 1 : 0; } fits_write_col(_fptr, TBIT, col, row, 1, size, dataChar.data(), &status); } aoflagger-v3.5.1/msio/fitsfile.h0000664000175000017500000002147614752462134014726 0ustar oleole/** @file * This is the header file for the FitsFile and FitsIOException class. * @author André Offringa */ #ifndef FITSFILE_H #define FITSFILE_H #include #include #include #include #include #include "../structures/types.h" /** * This class represents an exception that occurred during reading/writing of a * FITS file. * @see FitsFile */ class FitsIOException : public std::runtime_error { public: explicit FitsIOException(const std::string& description) : std::runtime_error(description) {} ~FitsIOException() noexcept {} }; /** * This class wraps the cfitsio library. It can be used to read or write a FITS * file with a (two dimensional) image. The fits file supports some other * things, like tables, which are not supported in this class. The FITS files * generated by the MeqTrees process can be read by this file. The FITS file can * store multiple images in one file. * * This class works closely together with the Image2D class. * @see Image2D */ class FitsFile { public: /** * Specifies how a FITS file should be opened */ enum FileMode { /** * Only open for reading */ ReadOnlyMode, /** * Open for reading and writing */ ReadWriteMode }; /** * The HDU is an entity inside a FITS file. This enum defines the possible * entity types. */ enum HDUType { /** * One or more two dimensional images */ ImageHDUType, /** * An ASCII table */ ASCIITableHDUType, /** * A binary table */ BinaryTableHDUType }; /** * This enum defines the possible image types in an image HDU. */ enum ImageType { /** * Image consisting of 8 bit integers. */ Int8ImageType, /** * Image consisting of 16 bit integers. */ Int16ImageType, /** * Image consisting of 32 bit integers. */ Int32ImageType, /** * Image consisting of 32 bit floats. */ Float32ImageType, /** * Image consisting of 64 bit doubles. */ Double64ImageType }; /** * Construct a new FitsFile class associated with the specified filename. * @param filename The file name of the fits file, to be opened with Open() or * created with Create(). */ explicit FitsFile(const std::string& filename); /** * Destructor. * @throws FitsIOException in case something failed due to an IO error. */ ~FitsFile(); /** * Open the file. * @param mode In which way the file should be opened. * @throws FitsIOException in case opening failed due to an IO error. */ void Open(FileMode mode = ReadOnlyMode); /** * Create a new FITS file. * @throws FitsIOException in case writing failed due to an IO error. */ void Create(); /** * Close the file, releasing resources. * @throws FitsIOException in case something happent due to an IO error. */ void Close(); /** * Determine whether the file is ready for reading and/or writing. * @return @c true if the file has been opened. */ bool IsOpen() const { return _isOpen; } /** * Retrieve the number of HDU blocks inside the file. * @return The number of HDU blocks inside the file. * @throws FitsIOException in case reading failed due to an IO error. */ int GetHDUCount(); /** * Retrieve the index of the current HDU block. * @return The index of the current HDU block. * @throws FitsIOException in case reading failed due to an IO error. */ int GetCurrentHDU(); /** * Retrieve the type of the current HDU block. * @return The type of the current HDU block. * @throws FitsIOException in case reading failed due to an IO error. */ enum HDUType GetCurrentHDUType(); /** * Start reading another HDU block, specified by its index. * @param hduNumber Index of the HDU block to move to. * @throws FitsIOException in case reading failed due to an IO error. */ void MoveToHDU(int hduNumber); /** * Retrieve what kind of image this HDU image block is. Only call this * method if GetCurrentHDUType() returned ImageHDUType. * @return The image type. * @throws FitsIOException in case reading failed due to an IO error. */ FitsFile::ImageType GetCurrentImageType(); /** * Retrieve the number of dimensions of an image. * @return The number of dimensions. * @see GetCurrentImageSize() * @throws FitsIOException in case reading failed due to an IO error. */ int GetCurrentImageDimensionCount(); /** * Retrieve the size of a specific dimension. * @param dimension The dimension to retrieve the size for (first dimension = * 0) * @return The size of the dimension. * @see GetCurrentImageDimensionCount() * @throws FitsIOException in case reading failed due to an IO error. */ long GetCurrentImageSize(int dimension); /** * Writes a new image HUD to the FITS file. Does not write the data itself, * only the headers. The WriteImage() call should be called next. * @param imageType Type of image * @param width Width of image * @param height Height of image * @throws FitsIOException in case writing failed due to an IO error. */ void AppendImageHUD(enum FitsFile::ImageType imageType, long width, long height); /** * Reads one image into a buffer. The image will be converted to @c long @c * doubles. * @param startPos This specifies where to start reading. If several images * are stored in the 3rd or 4th dimension, they can be read one by one by * starting at different start positions. * @param buffer The buffer where the image will be stored. * @param bufferSize Size of the buffer. Reading will stop once the buffer is * full. It makes sense to use buffer the size of the 1st x 2nd dimension. * @param nullValue What value should be used to represent null values. * @throws FitsIOException in case reading failed due to an IO error. */ void ReadCurrentImageData(long startPos, num_t* buffer, long bufferSize, long double nullValue = nan("Unset value")); /** * Writes an image to the FITS file. * @param startPos Where inside the image to start writing. * @param buffer Buffer containing the image data * @param bufferSize Size of the buffer * @param nullValue What value was used to represent null values. * @see ReadCurrentImageData * @throws FitsIOException in case writing failed due to an IO error. */ void WriteImage(long startPos, double* buffer, long bufferSize, double nullValue = nan("Unset value")); void WriteImage(long startPos, float* buffer, long bufferSize, float nullValue = nan("Unset value")); int GetKeywordCount(); bool HasKeyword(const std::string& keywordName); std::string GetKeyword(int keywordNumber); std::string GetKeywordValue(int keywordNumber); std::string GetKeywordValue(const std::string& keywordName); bool GetKeywordValue(const std::string& keywordName, std::string& value); std::string GetKeywordComment(int keywordNumber); int GetRowCount(); int GetColumnCount(); int GetColumnType(int colNumber); bool HasGroups(); int GetIntKeywordValue(int keywordNumber); int GetIntKeywordValue(const std::string& keywordName); double GetDoubleKeywordValue(int keywordNumber); double GetDoubleKeywordValue(const std::string& keywordName); int GetGroupCount(); int GetParameterCount(); long GetImageSize(); long GetGroupSize(); void ReadGroup(long groupIndex, long double* groupData); void ReadGroupData(long groupIndex, long double* groupData); void ReadGroupParameters(long groupIndex, long double* parametersData); void ReadTableCell(int row, int col, long double* output, size_t size); void ReadTableCell(int row, int col, double* output, size_t size); void ReadTableCell(int row, int col, bool* output, size_t size); void ReadTableCell(int row, int col, char* output); void WriteTableCell(int row, int col, double* data, size_t size); void WriteTableCell(int row, int col, const bool* data, size_t size); bool HasTableColumn(const std::string& columnName, int& columnIndex); int GetTableColumnIndex(const std::string& columnName); int GetTableColumnArraySize(int columnIndex); std::string GetTableDimensionName(int index); std::vector GetColumnDimensions(int columnIndex); long GetColumnDimensionSize(int columnIndex, int dimension); int GetGroupParameterIndex(const std::string& parameterName); int GetGroupParameterIndex(const std::string& parameterName, int number); bool HasGroupParameter(const std::string& parameterName); bool HasGroupParameter(const std::string& parameterName, int number); const std::string& Filename() const { return _filename; } private: const std::string _filename; fitsfile* _fptr; bool _isOpen; inline void CheckStatus(int status) const; inline void CheckOpen() const; }; #endif aoflagger-v3.5.1/msio/memorybaselinereader.h0000664000175000017500000000471414752462134017313 0ustar oleole#ifndef MEMORY_BASELINE_READER_H #define MEMORY_BASELINE_READER_H #include #include #include #include "../structures/antennainfo.h" #include "baselinereader.h" #include "../structures/image2d.h" #include "../structures/mask2d.h" class MemoryBaselineReader final : public BaselineReader { public: explicit MemoryBaselineReader(const std::string& msFile) : BaselineReader(msFile), _isRead(false), _areFlagsChanged(false) {} ~MemoryBaselineReader() { if (_areFlagsChanged) { WriteToMs(); } } void PrepareReadWrite(ProgressListener& progress) override; void PerformReadRequests(class ProgressListener& progress) override; void PerformFlagWriteRequests() override; void PerformDataWriteTask( [[maybe_unused]] std::vector realImages, [[maybe_unused]] std::vector imaginaryImages, [[maybe_unused]] size_t antenna1, [[maybe_unused]] size_t antenna2, [[maybe_unused]] size_t spectralWindow, [[maybe_unused]] size_t sequenceId) override { throw std::runtime_error( "The full mem reader can not write data back to file: use the indirect " "reader"); } static bool IsEnoughMemoryAvailable(uint64_t size); size_t GetMinRecommendedBufferSize(size_t /*threadCount*/) override { return 1; } size_t GetMaxRecommendedBufferSize(size_t /*threadCount*/) override { return 2; } bool IsModified() const override { return _areFlagsChanged; } void WriteToMs() override; private: void readSet(class ProgressListener& progress); void clear(); bool _isRead, _areFlagsChanged; class BaselineID { public: BaselineID(unsigned a1, unsigned a2, unsigned _spw, unsigned seqId) : antenna1(a1), antenna2(a2), spw(_spw), sequenceId(seqId) { if (antenna1 > antenna2) std::swap(antenna1, antenna2); } unsigned antenna1, antenna2, spw, sequenceId; bool operator<(const BaselineID& other) const { if (antenna1 < other.antenna1) { return true; } else if (antenna1 == other.antenna1) { if (antenna2 < other.antenna2) { return true; } else if (antenna2 == other.antenna2) { if (spw < other.spw) return true; else if (spw == other.spw) return sequenceId < other.sequenceId; } } return false; } }; std::map> _baselines; }; #endif // MEMORY_BASELINE_READER_H aoflagger-v3.5.1/msio/parmtable.h0000664000175000017500000002046614752462134015066 0ustar oleole#ifndef PARM_TABLE_H #define PARM_TABLE_H #include #include #include #include #include #include #include "../util/logger.h" #include "../structures/image2d.h" #include "../structures/timefrequencydata.h" class ParmTable { public: struct GainNameEntry { int index; int x, y; enum Component { Real, Imaginary } component; std::string antenna; GainNameEntry() : index(0), x(0), y(0), component(Real), antenna() {} GainNameEntry(const GainNameEntry& source) : index(source.index), x(source.x), y(source.y), component(source.component), antenna(source.antenna) {} void operator=(const GainNameEntry& source) { index = source.index; x = source.x; y = source.y; component = source.component; antenna = source.antenna; } }; explicit ParmTable(const std::string& path) : _path(path) { readNames(); } std::set GetAntennas() const { std::set antennas; for (GainNameEntryMap::const_iterator i = _nameEntries.begin(); i != _nameEntries.end(); ++i) { const GainNameEntry& entry = i->second; antennas.insert(entry.antenna); } return antennas; } TimeFrequencyData Read(const std::string& antenna) { Logger::Debug << "Reading antenna " << antenna << "\n"; // find the nameid's that we need to select const int r00 = FindEntry(0, 0, GainNameEntry::Real, antenna).index, r11 = FindEntry(1, 1, GainNameEntry::Real, antenna).index, i00 = FindEntry(0, 0, GainNameEntry::Imaginary, antenna).index, i11 = FindEntry(1, 1, GainNameEntry::Imaginary, antenna).index; Logger::Debug << "Names: r00=" << r00 << ", " << "r11=" << r11 << ", " << "i00=" << i00 << ", " << "i11=" << i11 << "\n"; casacore::Table table(_path); // Construct the images unsigned width, height; getImageDimensions(table, width, height, r00, r11, i00, i11); Image2DPtr xxReal = Image2D::CreateZeroImagePtr(width, height), yyReal = Image2D::CreateZeroImagePtr(width, height), xxImag = Image2D::CreateZeroImagePtr(width, height), yyImag = Image2D::CreateZeroImagePtr(width, height); // Read data casacore::ROScalarColumn nameIdColumn(table, "NAMEID"); casacore::ROScalarColumn startX(table, "STARTX"), startY(table, "STARTY"); casacore::ROArrayColumn values(table, "VALUES"); int xPos = 0, yPos = 0; // double currentX=startX(0); double currentY = startY(0); unsigned r00Count = 0, r11Count = 0, i00Count = 0, i11Count = 0; unsigned curXShape = 0; unsigned componentMatches = 0; for (unsigned row = 0; row < table.nrow(); ++row) { int nameId = nameIdColumn(row); if (nameId == r00 || nameId == r11 || nameId == i00 || nameId == i11) { Image2DPtr destImage; if (nameId == r00) { destImage = xxReal; ++r00Count; } else if (nameId == r11) { destImage = yyReal; ++r11Count; } else if (nameId == i00) { destImage = xxImag; ++i00Count; } else if (nameId == i11) { destImage = yyImag; ++i11Count; } const unsigned curYShape = values.shape(row)[1]; const unsigned xShape = values.shape(row)[0]; if (xShape > curXShape) curXShape = xShape; Logger::Debug << "Image has size " << xShape << " x " << curYShape << '\n'; const casacore::Array valueArray = values(row); casacore::Array::const_iterator vIter = valueArray.begin(); for (unsigned y = 0; y < curYShape; ++y) { for (unsigned x = 0; x < xShape; ++x) { destImage->SetValue(yPos + y, xPos + x, *vIter); ++vIter; } } ++componentMatches; if (componentMatches >= 4) { if (startY(row) < currentY) { xPos += curXShape; yPos = 0; curXShape = 0; } else { yPos += curYShape; } // currentX=startX(row); currentY = startY(row); componentMatches = 0; } } } Logger::Debug << "Counts: r00=" << r00Count << ", " << "r11=" << r11Count << ", " << "i00=" << i00Count << ", " << "i11=" << i11Count << "\n"; return TimeFrequencyData(aocommon::Polarization::XX, xxReal, xxImag, aocommon::Polarization::YY, yyReal, yyImag); } const GainNameEntry& FindEntry(int x, int y, enum GainNameEntry::Component c, const std::string& antenna) const { for (GainNameEntryMap::const_iterator i = _nameEntries.begin(); i != _nameEntries.end(); ++i) { const GainNameEntry& entry = i->second; if (entry.x == x && entry.y == y && entry.component == c && entry.antenna == antenna) { return entry; } } throw std::runtime_error("Entry not found"); } private: void readNames() { casacore::Table namesTable; if (_path.size() > 0 && *_path.rbegin() != '/') namesTable = casacore::Table(_path + "/NAMES"); else namesTable = casacore::Table(_path + "NAMES"); casacore::ROScalarColumn nameColumn(namesTable, "NAME"); for (unsigned i = 0; i != namesTable.nrow(); ++i) { std::string name = nameColumn(i); addName(i, name); } } void getImageDimensions(casacore::Table& table, unsigned& width, unsigned& height, int r00, int /*r11*/, int /*i00*/, int /*i11*/) { casacore::ROScalarColumn nameIdColumn(table, "NAMEID"); casacore::ROScalarColumn startX(table, "STARTX"), startY(table, "STARTY"); casacore::ROArrayColumn values(table, "VALUES"); int maxX = 0, yPos = 0; int maxY = 0; unsigned matches = 0; unsigned curXShape = 0; double currentX = startX(0), currentY = startY(0); for (unsigned row = 0; row < table.nrow(); ++row) { int nameId = nameIdColumn(row); if (nameId == r00) { const unsigned curYShape = values.shape(row)[1]; if (values.shape(row)[0] > (int)curXShape) curXShape = values.shape(row)[0]; if (startX(row) < currentX) throw std::runtime_error("Table is not correctly ordered"); yPos += curYShape; if (startY(row) < currentY) { maxX += curXShape; curXShape = 0; if (yPos > maxY) maxY = yPos; yPos = 0; } currentX = startX(row); currentY = startY(row); ++matches; } } maxX += curXShape; if (yPos > maxY) maxY = yPos; width = maxY; height = maxX; Logger::Debug << "Rows in table: " << table.nrow() << "\n" "Matching rows: " << matches << "\n" "Number of blocks: " << maxX << " x " << maxY << "\n" "Image size: " << width << " x " << height << "\n"; } void addName(unsigned index, const std::string& line) { size_t d1 = line.find(':'); std::string type = line.substr(0, d1); if (type == "Gain") { GainNameEntry entry; size_t d2 = line.find(':', d1 + 1), d3 = line.find(':', d2 + 1), d4 = line.find(':', d3 + 1); entry.index = index; entry.x = atoi(line.substr(d1 + 1, d2 - d1 - 1).c_str()); entry.y = atoi(line.substr(d2 + 1, d3 - d2 - 1).c_str()); std::string component = line.substr(d3 + 1, d4 - d3 - 1); if (component == "Real") entry.component = GainNameEntry::Real; else if (component == "Imag") entry.component = GainNameEntry::Imaginary; else throw std::runtime_error("Incorrect complex component type given"); entry.antenna = line.substr(d4 + 1); _nameEntries.insert(std::pair(index, entry)); } } std::string _path; typedef std::map GainNameEntryMap; GainNameEntryMap _nameEntries; }; #endif aoflagger-v3.5.1/msio/reorderedfilebuffer.h0000664000175000017500000000325714752462134017123 0ustar oleole#ifndef REORDERED_FILE_BUFFER_H #define REORDERED_FILE_BUFFER_H #include #include #include #include "../util/logger.h" class ReorderedFileBuffer { public: ReorderedFileBuffer(std::ofstream* stream, size_t maxSize) : _nextWritePos(0), _unflushedSize(0), _maxSize(maxSize), _stream(stream) {} ~ReorderedFileBuffer() { flush(); } void seekp(size_t offset) { _nextWritePos = offset; } void write(const char* data, size_t length) { _buffer.insert(BufferEntry(_nextWritePos, data, length)); _nextWritePos += length; _unflushedSize += length + sizeof(_nextWritePos) * 2; if (_unflushedSize > _maxSize) flush(); } void flush() { Logger::Debug << "Flushing reordered file buffer...\n"; for (std::set::const_iterator i = _buffer.begin(); i != _buffer.end(); ++i) { _stream->seekp(i->position, std::ios_base::beg); _stream->write(i->data.data(), i->data.size()); if (_stream->fail()) throw std::runtime_error( "Error: failed to write to reordered file! Check access rights and " "free disk space."); } _buffer.clear(); _unflushedSize = 0; } std::ofstream& stream() { return *_stream; } private: struct BufferEntry { BufferEntry(size_t pos_, const char* data_, size_t length_) : position(pos_), data(data_, data_ + length_) {} bool operator<(const BufferEntry& other) const { return position < other.position; } size_t position; std::vector data; }; std::set _buffer; size_t _nextWritePos; size_t _unflushedSize; size_t _maxSize; std::ofstream* _stream; }; #endif aoflagger-v3.5.1/msio/pngfile.h0000664000175000017500000000414614752462134014540 0ustar oleole/** @file * This is the header file for the PngFile class. * @author André Offringa */ #ifndef PNGFILE_H #define PNGFILE_H #include #include /** * This class wraps the libpng library. It can save an Image2D class to a .png * file. * @see Image2D */ class PngFile { public: /** * Construct a new png file with a filename, a width and a height. * @param filename Name of the png file. * @param width Width of the image * @param height Height of the image */ PngFile(const std::string& filename, size_t width, size_t height); /** * Destructor. */ ~PngFile(); /** * Start writing. * @throws IOException if something goes wrong. */ void BeginWrite(); /** * Closes the image. */ void Close(); /** * Returns the size of one pixel in bytes. * @return Size of one pixel in bytes. */ int PixelSize() const { return _pixelSize; } /** * Clears the entire image. * @param colorR Red background value. * @param colorG Green background value. * @param colorB Blue background value. * @param colorA Alfa background value. */ void Clear(int colorR = 255, int colorG = 255, int colorB = 255, int colorA = 255); /** * Sets a pixel in the image to a specific color. * @param x x-coordinate. * @param y y-coordinate. * @param colorR Red value. * @param colorG Green value. * @param colorB Blue value. * @param colorA Alfa value. */ void PlotPixel(size_t x, size_t y, int colorR, int colorG, int colorB, int colorA) { _row_pointers[y][x * _pixelSize] = colorR; _row_pointers[y][x * _pixelSize + 1] = colorG; _row_pointers[y][x * _pixelSize + 2] = colorB; _row_pointers[y][x * _pixelSize + 3] = colorA; } /** * Retrieve the array of row pointers. * @return an array of row pointers. */ png_bytep* RowPointers() const { return _row_pointers; } private: const std::string _filename; const size_t _width, _height; png_bytep* _row_pointers; png_structp _png_ptr; png_infop _info_ptr; FILE* _fp; const int _pixelSize; }; #endif aoflagger-v3.5.1/msio/msselection.h0000664000175000017500000000362414752462134015441 0ustar oleole#ifndef MS_SELECTION_H #define MS_SELECTION_H #include #include #include #include #include #include #include "../util/progress/progresslistener.h" class MSSelection { public: MSSelection(casacore::MeasurementSet& ms, const std::vector>& observationTimes, ProgressListener& progress) : _observationTimes(observationTimes), _ms(ms), _progress(progress) {} template void Process(Function function) { casacore::ScalarColumn timeColumn(_ms, "TIME"); casacore::ScalarColumn fieldIdColumn(_ms, "FIELD_ID"); double prevTime = -1.0; size_t prevFieldId = size_t(-1), sequenceId = size_t(-1), timeIndexInSequence = size_t(-1); for (size_t rowIndex = 0; rowIndex != _ms.nrow(); ++rowIndex) { _progress.OnProgress(rowIndex, _ms.nrow()); double time = timeColumn(rowIndex); bool newTime = time != prevTime; size_t fieldId = fieldIdColumn(rowIndex); if (fieldId != prevFieldId) { prevFieldId = fieldId; sequenceId++; newTime = true; } if (newTime) { const std::map& observationTimes = _observationTimes[sequenceId]; prevTime = time; auto elem = observationTimes.find(time); if (elem == observationTimes.end()) timeIndexInSequence = std::numeric_limits::max(); else timeIndexInSequence = elem->second; } if (timeIndexInSequence != std::numeric_limits::max()) { function(rowIndex, sequenceId, timeIndexInSequence); } } } private: std::vector> _observationTimes; casacore::MeasurementSet& _ms; ProgressListener& _progress; }; #endif aoflagger-v3.5.1/msio/spatialtimeloader.cpp0000664000175000017500000001263615063016106017143 0ustar oleole#include "spatialtimeloader.h" #include #include #include #include #include "../util/logger.h" SpatialTimeLoader::SpatialTimeLoader(MSMetaData& msMetaData) : _msMetaData(msMetaData) { const casacore::Table rawTable(_msMetaData.Path()); casacore::Block names(4); names[0] = "DATA_DESC_ID"; names[1] = "TIME"; names[2] = "ANTENNA1"; names[3] = "ANTENNA2"; _sortedTable.reset(new casacore::Table(rawTable.sort(names))); _channelCount = _msMetaData.FrequencyCount(0); _timestepsCount = _msMetaData.TimestepCount(); _antennaCount = _msMetaData.AntennaCount(); _polarizationCount = _msMetaData.PolarizationCount(); casacore::Block selectionNames(1); selectionNames[0] = "DATA_DESC_ID"; _tableIter.reset(new casacore::TableIterator( *_sortedTable, selectionNames, casacore::TableIterator::Ascending, casacore::TableIterator::NoSort)); } SpatialTimeLoader::~SpatialTimeLoader() {} TimeFrequencyData SpatialTimeLoader::Load(unsigned channelIndex, bool fringeStop) { const unsigned baselineCount = _antennaCount * (_antennaCount - 1) / 2; const casacore::Table table = _tableIter->table(); const casacore::ScalarColumn antenna1Column(table, "ANTENNA1"); const casacore::ScalarColumn antenna2Column(table, "ANTENNA2"); const casacore::ScalarColumn timeColumn(table, "TIME"); const casacore::ArrayColumn uvwColumn(table, "UVW"); const casacore::ArrayColumn flagColumn(table, "FLAG"); const casacore::ArrayColumn dataColumn(table, "DATA"); std::vector realImages(_polarizationCount), imagImages(_polarizationCount); std::vector masks(_polarizationCount); for (unsigned p = 0; p < _polarizationCount; ++p) { realImages[p] = Image2D::CreateUnsetImagePtr(_timestepsCount, baselineCount); imagImages[p] = Image2D::CreateUnsetImagePtr(_timestepsCount, baselineCount); masks[p] = Mask2D::CreateUnsetMaskPtr(_timestepsCount, baselineCount); } const ChannelInfo channelInfo = _msMetaData.GetBandInfo(0).channels[channelIndex]; unsigned timeIndex = 0; double lastTime = timeColumn(0); for (unsigned row = 0; row < table.nrow(); ++row) { const int a1 = antenna1Column(row), a2 = antenna2Column(row); const double time = timeColumn(row); if (time != lastTime) { timeIndex++; lastTime = time; } if (a1 != a2) { const casacore::Array data = dataColumn(row); const casacore::Array flags = flagColumn(row); const casacore::Array uvws = uvwColumn(row); casacore::Array::const_iterator i = data.begin(); casacore::Array::const_iterator fI = flags.begin(); casacore::Array::const_iterator uvwIter = uvws.begin(); ++uvwIter; ++uvwIter; const double wRotation = -channelInfo.MetersToLambda(*uvwIter) * M_PI * 2.0; const unsigned baselineIndex = baselineCount - (_antennaCount - a1) * (_antennaCount - a1 - 1) / 2 + a2 - a1 - 1; for (unsigned c = 0; c < _channelCount; ++c) { if (c == channelIndex) { Logger::Debug << "Reading timeIndex=" << timeIndex << ", baselineIndex=" << baselineIndex << ", a1=" << a1 << ", a2=" << a2 << ",w=" << wRotation << "\n"; for (unsigned p = 0; p < _polarizationCount; ++p) { double realValue = i->real(); double imagValue = i->imag(); if (fringeStop) { const double newRealValue = realValue * std::cos(wRotation) - imagValue * std::sin(wRotation); imagValue = realValue * std::sin(wRotation) + imagValue * std::cos(wRotation); realValue = newRealValue; } realImages[p]->SetValue(timeIndex, baselineIndex, realValue); imagImages[p]->SetValue(timeIndex, baselineIndex, imagValue); ++i; masks[p]->SetValue(timeIndex, baselineIndex, *fI); ++fI; } } else { for (unsigned p = 0; p < _polarizationCount; ++p) { ++i; ++fI; } } } } } const casacore::ROScalarColumn bandColumn(table, "DATA_DESC_ID"); const BandInfo band = _msMetaData.GetBandInfo(bandColumn(0)); TimeFrequencyData data; if (_polarizationCount == 4) { data = TimeFrequencyData::FromLinear( realImages[0], imagImages[0], realImages[1], imagImages[1], realImages[2], imagImages[2], realImages[3], imagImages[3]); data.SetIndividualPolarizationMasks(masks[0], masks[1], masks[2], masks[3]); } else if (_polarizationCount == 2) { data = TimeFrequencyData(aocommon::Polarization::XX, realImages[0], imagImages[0], aocommon::Polarization::YY, realImages[1], imagImages[1]); data.SetIndividualPolarizationMasks(masks[0], masks[1]); } else if (_polarizationCount == 1) { data = TimeFrequencyData(aocommon::Polarization::StokesI, realImages[0], imagImages[0]); data.SetGlobalMask(masks[0]); } else { throw std::runtime_error("Unknown number of polarizations!"); } return data; } aoflagger-v3.5.1/msio/averagingmsreader.cpp0000664000175000017500000004053215065216451017131 0ustar oleole#include "averagingmsreader.h" #include "../structures/mask2d.h" #include "../structures/msmetadata.h" #include "../structures/image2d.h" #include "../util/progress/progresslistener.h" #include #include #include #include using aocommon::Polarization; using aocommon::PolarizationEnum; AveragingMsReader::AveragingMsReader(const std::string& filename, const std::string& dataColumn) : _filename(filename), _dataColumn(dataColumn) { casacore::MeasurementSet ms(filename); _nBands = ms.spectralWindow().nrow(); const casacore::ScalarColumn fieldIdCol( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::FIELD_ID)); const casacore::ScalarColumn timeCol( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::TIME)); int curField = -1; for (size_t row = 0; row != ms.nrow(); ++row) { const int field = fieldIdCol(row); if (field != curField) { _sequenceStart.emplace_back(row); curField = field; } } _sequenceStart.emplace_back(ms.nrow()); _telescopeName = MSMetaData::GetTelescopeName(ms); } AveragingMsReader::Result AveragingMsReader::readSamples( BaselineIntegrationMode statType, size_t sequenceIndex, size_t bandIndex, bool includeAutos, bool includeFlags, bool freqDiff, ProgressListener& progress) { progress.OnStartTask("Read data & integrate baselines"); casacore::MeasurementSet ms(_filename); const casacore::ScalarColumn antenna1Col( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::ANTENNA1)); const casacore::ScalarColumn antenna2Col( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::ANTENNA2)); const casacore::ScalarColumn timeCol( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::TIME)); const casacore::ScalarColumn dataDescIdCol( ms, casacore::MeasurementSet::columnName( casacore::MeasurementSet::DATA_DESC_ID)); const casacore::ArrayColumn> dataColumn(ms, _dataColumn); const casacore::ArrayColumn flagColumn( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::FLAG)); casacore::IPosition dataShape = dataColumn.shape(0); size_t nPolarizations = dataShape[0], nChannels = dataShape[1], nTotal = freqDiff ? (nPolarizations * (nChannels - 1)) : nPolarizations * nChannels; casacore::Array> dataArray(dataShape); casacore::Array flagArray(dataShape); const aocommon::MultiBandData bands(ms.spectralWindow(), ms.dataDescription()); const size_t startRow = _sequenceStart[sequenceIndex]; const size_t endRow = _sequenceStart[sequenceIndex + 1]; double time = timeCol(startRow) - 1; std::vector statsData; std::vector times; size_t dataPos = 0; const size_t totalProgress = (endRow - startRow) * 104 / 100; for (size_t row = startRow; row != endRow; ++row) { size_t antenna1 = antenna1Col(row), antenna2 = antenna2Col(row); const bool baselineSelected = includeAutos || (antenna1 != antenna2); if (baselineSelected && bands.GetBandIndex(dataDescIdCol(row)) == bandIndex) { if (timeCol(row) != time) { dataPos = statsData.size(); statsData.resize(statsData.size() + nTotal); time = timeCol(row); times.emplace_back(time); } dataColumn.get(row, dataArray); flagColumn.get(row, flagArray); if (freqDiff) { for (size_t i = 0; i != nTotal; ++i) { const bool flag1 = flagArray.cbegin()[i]; const bool flag2 = flagArray.cbegin()[i + nPolarizations]; if ((!flag1 && !flag2) || includeFlags) { const std::complex val = dataArray.cbegin()[i] - dataArray.cbegin()[i + nPolarizations]; statsData[dataPos + i].Add(statType, val); } } } else { for (size_t i = 0; i != nTotal; ++i) { const bool flag = flagArray.cbegin()[i]; if (!flag || includeFlags) { const std::complex val = dataArray.cbegin()[i]; statsData[dataPos + i].Add(statType, val); } } } } progress.OnProgress(row - startRow, totalProgress); } progress.OnStartTask("Combining statistics"); Result result = makeResult(statType, statsData.data(), nPolarizations, freqDiff ? (nChannels - 1) : nChannels, times.size()); result.second->SetObservationTimes(times); fillBand(result, freqDiff, bands, bandIndex); progress.OnFinish(); return result; } AveragingMsReader::Result AveragingMsReader::readTimeDiff( BaselineIntegrationMode statType, size_t sequenceIndex, size_t bandIndex, bool includeAutos, bool includeFlags, ProgressListener& progress) { progress.OnStartTask("Read data & integrate baselines"); casacore::MeasurementSet ms(_filename); const casacore::ScalarColumn antenna1Col( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::ANTENNA1)); const casacore::ScalarColumn antenna2Col( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::ANTENNA2)); const casacore::ScalarColumn timeCol( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::TIME)); const casacore::ScalarColumn dataDescIdCol( ms, casacore::MeasurementSet::columnName( casacore::MeasurementSet::DATA_DESC_ID)); const casacore::ArrayColumn> dataColumn(ms, _dataColumn); const casacore::ArrayColumn flagColumn( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::FLAG)); casacore::IPosition dataShape = dataColumn.shape(0); size_t nPolarizations = dataShape[0], nChannels = dataShape[1], nTotal = nPolarizations * nChannels; casacore::Array> dataArray(dataShape); casacore::Array flagArray(dataShape); const aocommon::MultiBandData bands(ms.spectralWindow(), ms.dataDescription()); const size_t startRow = _sequenceStart[sequenceIndex]; const size_t endRow = _sequenceStart[sequenceIndex + 1]; double time = timeCol(startRow) - 1; std::vector statsData; std::vector times; size_t dataPos = 0; const size_t totalProgress = (endRow - startRow) * 104 / 100; using Baseline = std::pair; using TimeData = std::vector, bool>>; std::map previousTimeData, currentTimeData; for (size_t row = startRow; row != endRow; ++row) { size_t antenna1 = antenna1Col(row), antenna2 = antenna2Col(row); const bool baselineSelected = includeAutos || (antenna1 != antenna2); if (baselineSelected && bands.GetBandIndex(dataDescIdCol(row)) == bandIndex) { const std::pair baseline(antenna1, antenna2); if (timeCol(row) != time) { time = timeCol(row); times.emplace_back(time); if (times.size() > 1) { dataPos = statsData.size(); statsData.resize(statsData.size() + nTotal); previousTimeData = std::move(currentTimeData); currentTimeData.clear(); } } dataColumn.get(row, dataArray); flagColumn.get(row, flagArray); auto& curBaseline = currentTimeData[baseline]; curBaseline.resize(nTotal); for (size_t i = 0; i != nTotal; ++i) { const bool curFlag = flagArray.cbegin()[i]; const std::complex curVal = dataArray.cbegin()[i]; curBaseline[i].first = curVal; curBaseline[i].second = curFlag; } auto prevBaselinePtr = previousTimeData.find(baseline); if (times.size() > 1 && prevBaselinePtr != previousTimeData.end()) { for (size_t i = 0; i != nTotal; ++i) { const bool curFlag = flagArray.cbegin()[i]; const bool prevFlag = prevBaselinePtr->second[i].second; if ((!curFlag && !prevFlag) || includeFlags) { const std::complex val = prevBaselinePtr->second[i].first - dataArray.cbegin()[i]; statsData[dataPos + i].Add(statType, val); } } } } progress.OnProgress(row - startRow, totalProgress); } progress.OnStartTask("Combining statistics"); Result result = makeResult(statType, statsData.data(), nPolarizations, nChannels, times.size() - 1); fillBand(result, false, bands, bandIndex); for (size_t i = 0; i != times.size() - 1; ++i) times[i] = 0.5 * (times[i] + times[i + 1]); times.resize(times.size() - 1); result.second->SetObservationTimes(times); progress.OnFinish(); return result; } AveragingMsReader::Result AveragingMsReader::makeResult( BaselineIntegrationMode statType, const Statistic* statsData, size_t nPolarizations, size_t nChannels, size_t nTimes) { const size_t nTotal = nPolarizations * nChannels; std::vector images(nPolarizations); std::vector masks(nPolarizations); for (size_t p = 0; p != nPolarizations; ++p) { images[p] = Image2D::CreateUnsetImagePtr(nTimes, nChannels); masks[p] = Mask2D::CreateUnsetMaskPtr(nTimes, nChannels); } for (size_t y = 0; y != nChannels; ++y) { for (size_t x = 0; x != nTimes; ++x) { for (size_t p = 0; p != nPolarizations; ++p) { const Statistic& stat = statsData[x * nTotal + y * nPolarizations + p]; if (stat.count == 0) { images[p]->SetValue(x, y, 0.0); masks[p]->SetValue(x, y, true); } else { const double val = stat.Calculate(statType); images[p]->SetValue(x, y, val); masks[p]->SetValue(x, y, false); } } } } std::vector pols; if (nPolarizations == 4) pols = std::vector{Polarization::XX, Polarization::XY, Polarization ::YX, Polarization::YY}; else if (nPolarizations == 2) pols = std::vector{Polarization::XX, Polarization::YY}; else pols.emplace_back(Polarization::StokesI); std::vector tfs; for (size_t p = 0; p != nPolarizations; ++p) { tfs.emplace_back(TimeFrequencyData::AmplitudePart, pols[p], images[p]); tfs.back().SetGlobalMask(masks[p]); } Result result; if (nPolarizations == 4) result.first = TimeFrequencyData::MakeFromPolarizationCombination( tfs[0], tfs[1], tfs[2], tfs[3]); else if (nPolarizations == 2) result.first = TimeFrequencyData::MakeFromPolarizationCombination(tfs[0], tfs[1]); else result.first = std::move(tfs[0]); result.second.reset(new TimeFrequencyMetaData()); return result; } void AveragingMsReader::fillBand(AveragingMsReader::Result& result, bool freqDiff, const aocommon::MultiBandData& bands, size_t bandIndex) { BandInfo bandInfo; bandInfo.windowIndex = 0; if (freqDiff) { bandInfo.channels.resize(bands[bandIndex].ChannelCount() - 1); for (size_t i = 0; i != bands[bandIndex].ChannelCount() - 1; ++i) { bandInfo.channels[i].frequencyIndex = i; bandInfo.channels[i].frequencyHz = 0.5 * (bands[bandIndex].Channel(i).Frequency() + bands[bandIndex].Channel(i + 1).Frequency()); bandInfo.channels[i].effectiveBandWidthHz = bands[bandIndex].Channel(i).Width() + bands[bandIndex].Channel(i + 1).Width(); } } else { bandInfo.channels.resize(bands[bandIndex].ChannelCount()); for (size_t i = 0; i != bands[bandIndex].ChannelCount(); ++i) { bandInfo.channels[i].frequencyIndex = i; bandInfo.channels[i].frequencyHz = bands[bandIndex].Channel(i).Frequency(); bandInfo.channels[i].effectiveBandWidthHz = bands[bandIndex].Channel(i).Width(); } } result.second->SetBand(bandInfo); } void AveragingMsReader::StoreFlags(const std::vector& flags, BaselineIntegrationDifferencing diffType, size_t sequenceIndex, size_t bandIndex, bool includeAutos) { std::vector appliedFlags; switch (diffType) { case BaselineIntegrationDifferencing::NoDifference: appliedFlags = flags; break; case BaselineIntegrationDifferencing::TimeDifference: for (size_t i = 0; i != flags.size(); ++i) { const Mask2DCPtr input = flags[i]; Mask2DPtr mask = Mask2D::CreateUnsetMaskPtr(input->Width() + 1, input->Height()); for (size_t y = 0; y != input->Height(); ++y) { mask->SetValue(0, y, input->Value(0, y)); for (size_t x = 1; x != input->Width(); ++x) { mask->SetValue(x, y, input->Value(x - 1, y) || input->Value(x, y)); } mask->SetValue(input->Width(), y, input->Value(input->Width() - 1, y)); } appliedFlags.emplace_back(std::move(mask)); } break; case BaselineIntegrationDifferencing::FrequencyDifference: for (size_t i = 0; i != flags.size(); ++i) { const Mask2DCPtr input = flags[i]; Mask2DPtr mask = Mask2D::CreateUnsetMaskPtr(input->Width(), input->Height() + 1); for (size_t x = 0; x != input->Width(); ++x) mask->SetValue(x, 0, input->Value(x, 0)); for (size_t y = 1; y != input->Height(); ++y) { for (size_t x = 0; x != input->Width(); ++x) { mask->SetValue(x, y, input->Value(x, y - 1) || input->Value(x, y)); } } for (size_t x = 0; x != input->Width(); ++x) mask->SetValue(x, input->Height(), input->Value(x, input->Height() - 1)); appliedFlags.emplace_back(std::move(mask)); } break; } storeFlags(appliedFlags, sequenceIndex, bandIndex, includeAutos); } void AveragingMsReader::storeFlags(const std::vector& flags, size_t sequenceIndex, size_t bandIndex, bool includeAutos) { casacore::MeasurementSet ms(_filename, casacore::MeasurementSet::TableOption::Update); const casacore::ScalarColumn antenna1Col( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::ANTENNA1)); const casacore::ScalarColumn antenna2Col( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::ANTENNA2)); const casacore::ScalarColumn timeCol( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::TIME)); const casacore::ScalarColumn dataDescIdCol( ms, casacore::MeasurementSet::columnName( casacore::MeasurementSet::DATA_DESC_ID)); casacore::ArrayColumn flagColumn( ms, casacore::MeasurementSet::columnName(casacore::MeasurementSet::FLAG)); casacore::IPosition dataShape = flagColumn.shape(0); size_t nPolarizations = dataShape[0], nChannels = dataShape[1]; if (nPolarizations != flags.size()) throw std::runtime_error( "Invalid nr of masks specified in call to MSStatReader::storeFlags()"); casacore::Array flagArray(dataShape); const aocommon::MultiBandData bands(ms.spectralWindow(), ms.dataDescription()); size_t startRow = _sequenceStart[sequenceIndex], endRow = _sequenceStart[sequenceIndex + 1]; double time = timeCol(startRow) - 1; size_t timeIndex = 0; for (size_t row = startRow; row != endRow; ++row) { size_t antenna1 = antenna1Col(row), antenna2 = antenna2Col(row); const bool baselineSelected = includeAutos || (antenna1 != antenna2); if (baselineSelected && bands.GetBandIndex(dataDescIdCol(row)) == bandIndex) { const std::pair baseline(antenna1, antenna2); if (timeCol(row) != time) { time = timeCol(row); ++timeIndex; } flagColumn.get(row, flagArray); auto iter = flagArray.cbegin(); for (size_t ch = 0; ch != nChannels; ++ch) { for (size_t p = 0; p != nPolarizations; ++p) { *iter = *iter || flags[p]->Value(timeIndex - 1, ch); ++iter; } } flagColumn.put(row, flagArray); } } } aoflagger-v3.5.1/msio/reorderingbaselinereader.h0000664000175000017500000000724314752462134020143 0ustar oleole#ifndef MSIO_REORDERING_BASELINE_READER_H_ #define MSIO_REORDERING_BASELINE_READER_H_ #include #include #include #include #include #include "baselinereader.h" #include "directbaselinereader.h" class ReorderingBaselineReader : public BaselineReader { public: explicit ReorderingBaselineReader(const std::string& msFile); ~ReorderingBaselineReader(); bool IsModified() const override { return reordered_data_files_have_changed_ || reordered_flag_files_have_changed_; } void WriteToMs() override; void PrepareReadWrite(ProgressListener& progress) override; virtual void PerformReadRequests( class ProgressListener& progress) final override; virtual void PerformFlagWriteRequests() final override; virtual void PerformDataWriteTask(std::vector _realImages, std::vector _imaginaryImages, size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId) final override; virtual size_t GetMinRecommendedBufferSize( size_t /*threadCount*/) final override { return 1; } virtual size_t GetMaxRecommendedBufferSize( size_t /*threadCount*/) final override { return 2; } void SetReadUVW(bool readUVW) { read_uvw_ = readUVW; } private: struct ReorderInfo { std::unique_ptr dataFile; std::unique_ptr flagFile; }; struct UpdateInfo { std::unique_ptr dataFile; std::unique_ptr flagFile; }; class SeqIndexLookupTable { public: SeqIndexLookupTable(size_t antennaCount, size_t spectralWindowCount, size_t sequenceCount) : _antennaCount(antennaCount), _table(sequenceCount) { size_t maxBaselineCount = antennaCount * antennaCount; for (size_t i = 0; i != sequenceCount; ++i) { std::vector>& spwTable = _table[i]; spwTable.resize(spectralWindowCount); for (size_t j = 0; j != spectralWindowCount; ++j) { std::vector& baselTable = spwTable[j]; baselTable.resize(maxBaselineCount); } } } size_t& Value(size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId) { return _table[sequenceId][spectralWindow] [antenna1 * _antennaCount + antenna2]; } private: size_t _antennaCount; std::vector>> _table; }; void reorderMS(class ProgressListener& progress); void reorderFull(class ProgressListener& progress); void makeLookupTables(size_t& fileSize); void updateOriginalMSData(class ProgressListener& progress); void updateOriginalMSFlags(class ProgressListener& progress); void performFlagWriteTask(std::vector flags, unsigned antenna1, unsigned antenna2, unsigned spw, unsigned sequenceId); template void updateOriginalMS(class ProgressListener& progress); void removeTemporaryFiles(); static void preAllocate(const std::string& filename, size_t fileSize); DirectBaselineReader direct_reader_; std::unique_ptr sequence_index_table_; std::vector file_positions_; std::string data_filename_; std::string flag_filename_; std::string meta_filename_; bool ms_is_reordered_; bool remove_reordered_files_; bool reordered_data_files_have_changed_; bool reordered_flag_files_have_changed_; bool read_uvw_; }; #endif // MSIO_REORDERING_BASELINE_READER_H_ aoflagger-v3.5.1/msio/reorderingbaselinereader.cpp0000664000175000017500000005344614752462134020504 0ustar oleole#include "reorderingbaselinereader.h" #include "../structures/timefrequencydata.h" #include "../util/logger.h" #include "../util/stopwatch.h" #include "../util/progress/dummyprogresslistener.h" #include "reorderedfilebuffer.h" #include "msselection.h" #include #include #include #include #include #include #include #include ReorderingBaselineReader::ReorderingBaselineReader(const std::string& msFile) : BaselineReader(msFile), direct_reader_(msFile), sequence_index_table_(), ms_is_reordered_(false), remove_reordered_files_(false), reordered_data_files_have_changed_(false), reordered_flag_files_have_changed_(false), read_uvw_(false) { // In order to use multiple readers at the same time the temporary files need // unique names. Use the address of the object to generate a unique prefix. const std::string uid = std::to_string(reinterpret_cast(this)); data_filename_ = uid + "-aoflagger-data.tmp"; flag_filename_ = uid + "-aoflagger-flag.tmp"; meta_filename_ = uid + "-ao-msinfo.tmp"; } ReorderingBaselineReader::~ReorderingBaselineReader() { WriteToMs(); removeTemporaryFiles(); } void ReorderingBaselineReader::WriteToMs() { DummyProgressListener dummy; if (reordered_data_files_have_changed_) updateOriginalMSData(dummy); if (reordered_flag_files_have_changed_) updateOriginalMSFlags(dummy); } void ReorderingBaselineReader::PrepareReadWrite(ProgressListener& progress) { if (!ms_is_reordered_) { reorderMS(progress); } } void ReorderingBaselineReader::PerformReadRequests( class ProgressListener& progress) { initializeMeta(); PrepareReadWrite(dummy_progress_); _results.clear(); for (size_t i = 0; i < _readRequests.size(); ++i) { const ReadRequest request = _readRequests[i]; _results.push_back(Result()); const size_t width = ObservationTimes(request.sequenceId).size(); for (size_t p = 0; p < Polarizations().size(); ++p) { if (ReadData()) { _results[i]._realImages.push_back(Image2D::CreateZeroImagePtr( width, MetaData().FrequencyCount(request.spectralWindow))); _results[i]._imaginaryImages.push_back(Image2D::CreateZeroImagePtr( width, MetaData().FrequencyCount(request.spectralWindow))); } if (ReadFlags()) { // The flags should be initialized to true, as a baseline might // miss some time scans that other baselines do have, and these // should be flagged. _results[i]._flags.push_back(Mask2D::CreateSetMaskPtr( width, MetaData().FrequencyCount(request.spectralWindow))); } } if (read_uvw_) { _results[i]._uvw = direct_reader_.ReadUVW(request.antenna1, request.antenna2, request.spectralWindow, request.sequenceId); } else { _results[i]._uvw.clear(); for (unsigned j = 0; j < width; ++j) _results[i]._uvw.emplace_back(0.0, 0.0, 0.0); } std::ifstream dataFile(data_filename_, std::ifstream::binary); std::ifstream flagFile(flag_filename_, std::ifstream::binary); const size_t index = sequence_index_table_->Value( request.antenna1, request.antenna2, request.spectralWindow, request.sequenceId); const size_t filePos = file_positions_[index]; dataFile.seekg(filePos * (sizeof(float) * 2), std::ios_base::beg); flagFile.seekg(filePos * sizeof(bool), std::ios_base::beg); const size_t bufferSize = MetaData().FrequencyCount(request.spectralWindow) * Polarizations().size(); for (size_t x = 0; x < width; ++x) { std::vector dataBuffer(bufferSize * 2); std::vector flagBuffer(bufferSize); dataFile.read((char*)&dataBuffer[0], bufferSize * sizeof(float) * 2); size_t dataBufferPtr = 0; flagFile.read((char*)&flagBuffer[0], bufferSize * sizeof(bool)); size_t flagBufferPtr = 0; for (size_t f = 0; f < MetaData().FrequencyCount(request.spectralWindow); ++f) { for (size_t p = 0; p < Polarizations().size(); ++p) { _results[i]._realImages[p]->SetValue(x, f, dataBuffer[dataBufferPtr]); ++dataBufferPtr; _results[i]._imaginaryImages[p]->SetValue(x, f, dataBuffer[dataBufferPtr]); ++dataBufferPtr; _results[i]._flags[p]->SetValue(x, f, flagBuffer[flagBufferPtr]); ++flagBufferPtr; } } } } _readRequests.clear(); progress.OnFinish(); } void ReorderingBaselineReader::PerformFlagWriteRequests() { for (size_t i = 0; i != _writeRequests.size(); ++i) { const FlagWriteRequest request = _writeRequests[i]; performFlagWriteTask(request.flags, request.antenna1, request.antenna2, request.spectralWindow, request.sequenceId); } _writeRequests.clear(); } void ReorderingBaselineReader::reorderMS(ProgressListener& progress) { initializeMeta(); progress.OnStartTask("Reordering measurement set"); const std::filesystem::path path(meta_filename_); bool reorderRequired = true; if (std::filesystem::exists(path)) { std::ifstream str(path.string().c_str()); std::string name; std::getline(str, name); if (std::filesystem::equivalent(std::filesystem::path(name), MetaData().Path())) { Logger::Debug << "Measurement set has already been reordered; using old " "temporary files.\n"; reorderRequired = false; ms_is_reordered_ = true; remove_reordered_files_ = false; reordered_data_files_have_changed_ = false; reordered_flag_files_have_changed_ = false; } } if (reorderRequired) { reorderFull(progress); std::ofstream str(path.string().c_str()); str << MetaData().Path() << '\n'; } else { size_t fileSize; makeLookupTables(fileSize); } } void ReorderingBaselineReader::makeLookupTables(size_t& fileSize) { std::vector sequences = MetaData().GetSequences(); const size_t antennaCount = MetaData().AntennaCount(), polarizationCount = Polarizations().size(), bandCount = MetaData().BandCount(), sequencesPerBaselineCount = MetaData().SequenceCount(); sequence_index_table_.reset(new SeqIndexLookupTable( antennaCount, bandCount, sequencesPerBaselineCount)); fileSize = 0; for (size_t i = 0; i < sequences.size(); ++i) { // Initialize look-up table to get index into Sequence-array quickly const MSMetaData::Sequence& s = sequences[i]; sequence_index_table_->Value(s.antenna1, s.antenna2, s.spw, s.sequenceId) = i; // Initialize look-up table to go from sequence array to file position. Is // in samples, so multiple times sizeof(bool) or ..(float)) for exact // position. file_positions_.push_back(fileSize); fileSize += ObservationTimes(s.sequenceId).size() * MetaData().FrequencyCount(s.spw) * polarizationCount; } } void ReorderingBaselineReader::preAllocate(const std::string& filename, size_t fileSize) { Logger::Debug << "Pre-allocating " << (fileSize / (1024 * 1024)) << " MB...\n"; const int fd = open(filename.c_str(), O_WRONLY | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR); if (fd < 0) { std::ostringstream s; s << "Error while opening file '" << filename << "', check access rights and free space"; throw std::runtime_error(s.str()); } #if defined(HAVE_POSIX_FALLOCATE) const int allocResult = posix_fallocate(fd, 0, fileSize); close(fd); if (allocResult != 0) { Logger::Warn << "Could not allocate temporary file '" << filename << "': posix_fallocate returned " << allocResult << ".\n" "Tried to allocate " << (fileSize / (1024 * 1024)) << " MB.\n" "Disk could be full or filesystem could not support fallocate.\n"; } #else close(fd); Logger::Warn << "Compiled without posix_fallocate() support: skipping " "pre-allocation.\n"; #endif } void ReorderingBaselineReader::reorderFull(ProgressListener& progressListener) { const Stopwatch watch(true); casacore::MeasurementSet ms = OpenMS(); casacore::ArrayColumn flagColumn(ms, "FLAG"); casacore::ScalarColumn dataDescIdColumn(ms, "DATA_DESC_ID"); casacore::ScalarColumn antenna1Column(ms, "ANTENNA1"); casacore::ScalarColumn antenna2Column(ms, "ANTENNA2"); casacore::ArrayColumn dataColumn(ms, DataColumnName()); if (ms.nrow() == 0) throw std::runtime_error("Measurement set is empty (zero rows)"); std::vector dataIdToSpw; MetaData().GetDataDescToBandVector(dataIdToSpw); size_t fileSize; makeLookupTables(fileSize); Logger::Debug << "Opening temporary files.\n"; ReorderInfo reorderInfo; preAllocate(data_filename_, fileSize * sizeof(float) * 2); reorderInfo.dataFile.reset(new std::ofstream( data_filename_, std::ofstream::binary | std::ios_base::in | std::ios_base::out)); if (reorderInfo.dataFile->fail()) throw std::runtime_error( "Error: failed to open temporary data files for writing! Check access " "rights and free disk space."); preAllocate(flag_filename_, fileSize * sizeof(bool)); reorderInfo.flagFile.reset(new std::ofstream( flag_filename_, std::ofstream::binary | std::ios_base::in | std::ios_base::out)); if (reorderInfo.flagFile->fail()) throw std::runtime_error( "Error: failed to open temporary data files for writing! Check access " "rights and free disk space."); Logger::Debug << "Reordering data set...\n"; const size_t bufferMem = std::min( aocommon::system::TotalMemory() / 10, 1024l * 1024l * 1024l); ReorderedFileBuffer dataFile(reorderInfo.dataFile.get(), bufferMem); ReorderedFileBuffer flagFile(reorderInfo.flagFile.get(), bufferMem / 8); std::vector writeFilePositions = file_positions_; std::vector timePositions(file_positions_.size(), size_t(-1)); size_t polarizationCount = Polarizations().size(); MSSelection msSelection(ms, ObservationTimesPerSequence(), progressListener); msSelection.Process( [&](size_t rowIndex, size_t sequenceId, size_t timeIndexInSequence) { size_t antenna1 = antenna1Column(rowIndex), antenna2 = antenna2Column(rowIndex), spw = dataIdToSpw[dataDescIdColumn(rowIndex)], channelCount = MetaData().FrequencyCount(spw), arrayIndex = sequence_index_table_->Value(antenna1, antenna2, spw, sequenceId), sampleCount = channelCount * polarizationCount; size_t& filePos = writeFilePositions[arrayIndex]; size_t& timePos = timePositions[arrayIndex]; casacore::Array data = dataColumn(rowIndex); casacore::Array flag = flagColumn(rowIndex); dataFile.seekp(filePos * (sizeof(float) * 2)); flagFile.seekp(filePos * sizeof(bool)); // If this baseline missed some time steps, pad the files // (we can't just skip over, because the flags should be set to true) ++timePos; while (timePos < timeIndexInSequence) { const std::vector nullData(sampleCount * 2, 0.0); const std::vector nullFlags(sampleCount, (char)true); dataFile.write(reinterpret_cast(&*nullData.begin()), sampleCount * 2 * sizeof(float)); flagFile.write(reinterpret_cast(&*nullFlags.begin()), sampleCount * sizeof(bool)); ++timePos; filePos += sampleCount; } dataFile.write(reinterpret_cast(&*data.cbegin()), sampleCount * 2 * sizeof(float)); flagFile.write(reinterpret_cast(&*flag.cbegin()), sampleCount * sizeof(bool)); filePos += sampleCount; }); const uint64_t dataSetSize = (uint64_t)fileSize * (uint64_t)(sizeof(float) * 2 + sizeof(bool)); Logger::Debug << "Done reordering data set of " << dataSetSize / (1024 * 1024) << " MB in " << watch.Seconds() << " s (" << (long double)dataSetSize / (1024.0L * 1024.0L * watch.Seconds()) << " MB/s)\n"; ms_is_reordered_ = true; remove_reordered_files_ = true; reordered_data_files_have_changed_ = false; reordered_flag_files_have_changed_ = false; } void ReorderingBaselineReader::removeTemporaryFiles() { if (ms_is_reordered_ && remove_reordered_files_) { std::filesystem::remove(meta_filename_); std::filesystem::remove(data_filename_); std::filesystem::remove(flag_filename_); Logger::Debug << "Temporary files removed.\n"; } ms_is_reordered_ = false; remove_reordered_files_ = false; reordered_data_files_have_changed_ = false; reordered_flag_files_have_changed_ = false; } void ReorderingBaselineReader::PerformDataWriteTask( std::vector _realImages, std::vector _imaginaryImages, size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId) { initializeMeta(); Logger::Debug << "Performing data write task with indirect baseline reader...\n"; const size_t polarizationCount = Polarizations().size(); if (_realImages.size() != polarizationCount || _imaginaryImages.size() != polarizationCount) throw std::runtime_error( "PerformDataWriteTask: input format did not match number of " "polarizations in measurement set"); for (size_t i = 1; i < _realImages.size(); ++i) { if (_realImages[0]->Width() != _realImages[i]->Width() || _realImages[0]->Height() != _realImages[i]->Height() || _realImages[0]->Width() != _imaginaryImages[i]->Width() || _realImages[0]->Height() != _imaginaryImages[i]->Height()) throw std::runtime_error( "PerformDataWriteTask: width and/or height of input images did not " "match"); } PrepareReadWrite(dummy_progress_); const size_t width = _realImages[0]->Width(); const size_t bufferSize = MetaData().FrequencyCount(spectralWindow) * Polarizations().size(); std::ofstream dataFile(data_filename_, std::ofstream::binary | std::ios_base::in | std::ios_base::out); const size_t index = sequence_index_table_->Value(antenna1, antenna2, spectralWindow, sequenceId); const size_t filePos = file_positions_[index]; dataFile.seekp(filePos * (sizeof(float) * 2), std::ios_base::beg); std::vector dataBuffer(bufferSize * 2); for (size_t x = 0; x < width; ++x) { size_t dataBufferPtr = 0; for (size_t f = 0; f < MetaData().FrequencyCount(spectralWindow); ++f) { for (size_t p = 0; p < Polarizations().size(); ++p) { dataBuffer[dataBufferPtr] = _realImages[p]->Value(x, f); ++dataBufferPtr; dataBuffer[dataBufferPtr] = _imaginaryImages[p]->Value(x, f); ++dataBufferPtr; } } dataFile.write(reinterpret_cast(&dataBuffer[0]), bufferSize * sizeof(float) * 2); if (dataFile.bad()) throw std::runtime_error( "Error: failed to update temporary data files! Check access rights " "and free disk space."); } reordered_data_files_have_changed_ = true; Logger::Debug << "Done writing.\n"; } void ReorderingBaselineReader::performFlagWriteTask( std::vector flags, unsigned antenna1, unsigned antenna2, unsigned spw, unsigned sequenceId) { initializeMeta(); const unsigned polarizationCount = Polarizations().size(); if (flags.size() != polarizationCount) throw std::runtime_error( "PerformDataWriteTask: input format did not match number of " "polarizations in measurement set"); for (size_t i = 1; i < flags.size(); ++i) { if (flags[0]->Width() != flags[i]->Width() || flags[0]->Height() != flags[i]->Height()) throw std::runtime_error( "PerformDataWriteTask: width and/or height of input images did not " "match"); } PrepareReadWrite(dummy_progress_); const size_t width = flags[0]->Width(); const size_t bufferSize = MetaData().FrequencyCount(spw) * Polarizations().size(); std::ofstream flagFile(flag_filename_, std::ofstream::binary | std::ios_base::in | std::ios_base::out); const size_t index = sequence_index_table_->Value(antenna1, antenna2, spw, sequenceId); const size_t filePos = file_positions_[index]; flagFile.seekp(filePos * (sizeof(bool)), std::ios_base::beg); const std::unique_ptr flagBuffer(new bool[bufferSize]); for (size_t x = 0; x < width; ++x) { size_t flagBufferPtr = 0; for (size_t f = 0; f < MetaData().FrequencyCount(spw); ++f) { for (size_t p = 0; p < polarizationCount; ++p) { flagBuffer[flagBufferPtr] = flags[p]->Value(x, f); ++flagBufferPtr; } } flagFile.write(reinterpret_cast(flagBuffer.get()), bufferSize * sizeof(bool)); if (flagFile.bad()) throw std::runtime_error( "Error: failed to update temporary flag files! Check access rights " "and free disk space."); } reordered_flag_files_have_changed_ = true; } template void ReorderingBaselineReader::updateOriginalMS(ProgressListener& progress) { casacore::MeasurementSet ms = OpenMS(); if (UpdateData || UpdateFlags) { ms.reopenRW(); } const casacore::ScalarColumn timeColumn(ms, "TIME"); const casacore::ScalarColumn antenna1Column(ms, "ANTENNA1"); const casacore::ScalarColumn antenna2Column(ms, "ANTENNA2"); const casacore::ScalarColumn fieldIdColumn(ms, "FIELD_ID"); const casacore::ScalarColumn dataDescIdColumn(ms, "DATA_DESC_ID"); casacore::ArrayColumn flagColumn(ms, "FLAG"); casacore::ArrayColumn dataColumn(ms, DataColumnName()); const std::vector sequences = MetaData().GetSequences(); std::vector dataIdToSpw; MetaData().GetDataDescToBandVector(dataIdToSpw); const size_t polarizationCount = Polarizations().size(); Logger::Debug << "Opening updated files\n"; UpdateInfo updateInfo; if (UpdateData) { updateInfo.dataFile.reset( new std::ifstream(data_filename_, std::ifstream::binary)); if (updateInfo.dataFile->fail()) throw std::runtime_error("Failed to open temporary data file"); } if (UpdateFlags) { updateInfo.flagFile.reset( new std::ifstream(flag_filename_, std::ifstream::binary)); if (updateInfo.flagFile->fail()) throw std::runtime_error("Failed to open temporary flag file"); } std::vector updatedFilePos = file_positions_; std::vector timePositions(updatedFilePos.size(), size_t(-1)); MSSelection msSelection(ms, ObservationTimesPerSequence(), progress); msSelection.Process([&](size_t rowIndex, size_t sequenceId, size_t timeIndexInSequence) { size_t antenna1 = antenna1Column(rowIndex), antenna2 = antenna2Column(rowIndex), spw = dataIdToSpw[dataDescIdColumn(rowIndex)], channelCount = MetaData().FrequencyCount(spw), arrayIndex = sequence_index_table_->Value(antenna1, antenna2, spw, sequenceId), sampleCount = channelCount * polarizationCount; size_t& filePos = updatedFilePos[arrayIndex]; size_t& timePos = timePositions[arrayIndex]; const casacore::IPosition shape(2, polarizationCount, channelCount); // Skip over samples in the temporary files that are missing in the // measurement set ++timePos; while (timePos < timeIndexInSequence) { filePos += sampleCount; ++timePos; } if (UpdateData) { casacore::Array data(shape); std::ifstream& dataFile = *updateInfo.dataFile; dataFile.seekg(filePos * (sizeof(float) * 2), std::ios_base::beg); dataFile.read(reinterpret_cast(&*data.cbegin()), sampleCount * 2 * sizeof(float)); if (dataFile.fail()) throw std::runtime_error("Error: failed to read temporary data files!"); dataColumn.basePut(rowIndex, data); } if (UpdateFlags) { casacore::Array flagArray(shape); std::ifstream& flagFile = *updateInfo.flagFile; flagFile.seekg(filePos * sizeof(bool), std::ios_base::beg); flagFile.read(reinterpret_cast(&*flagArray.cbegin()), sampleCount * sizeof(bool)); if (flagFile.fail()) throw std::runtime_error("Error: failed to read temporary flag files!"); flagColumn.basePut(rowIndex, flagArray); } filePos += sampleCount; }); Logger::Debug << "Freeing the data\n"; // Close the files updateInfo.dataFile.reset(); updateInfo.flagFile.reset(); if (UpdateData) Logger::Debug << "Done updating measurement set data\n"; if (UpdateFlags) Logger::Debug << "Done updating measurement set flags\n"; } void ReorderingBaselineReader::updateOriginalMSData( ProgressListener& progress) { Logger::Debug << "Data was changed, need to update the original MS...\n"; updateOriginalMS(progress); reordered_data_files_have_changed_ = false; } void ReorderingBaselineReader::updateOriginalMSFlags( ProgressListener& progress) { const Stopwatch watch(true); Logger::Debug << "Flags were changed, need to update the original MS...\n"; updateOriginalMS(progress); reordered_flag_files_have_changed_ = false; Logger::Debug << "Storing flags toke: " << watch.ToString() << '\n'; } aoflagger-v3.5.1/msio/averagingmsreader.h0000664000175000017500000001004615065216451016573 0ustar oleole#ifndef MS_STAT_READER_H #define MS_STAT_READER_H #include #include #include #include #include "../structures/timefrequencydata.h" #include "../structures/timefrequencymetadata.h" #include "../imagesets/msoptions.h" namespace aocommon { class MultiBandData; } class AveragingMsReader { public: AveragingMsReader(const std::string& filename, const std::string& dataColumn); size_t NSequences() const { return _sequenceStart.size() - 1; } size_t NBands() const { return _nBands; } typedef std::pair Result; Result Read(BaselineIntegrationMode statType, BaselineIntegrationDifferencing diffType, size_t sequenceIndex, size_t bandIndex, bool includeAutos, bool includeFlags, class ProgressListener& progress) { switch (diffType) { default: case BaselineIntegrationDifferencing::NoDifference: return readSamples(statType, sequenceIndex, bandIndex, includeAutos, includeFlags, false, progress); case BaselineIntegrationDifferencing::TimeDifference: return readTimeDiff(statType, sequenceIndex, bandIndex, includeAutos, includeFlags, progress); case BaselineIntegrationDifferencing::FrequencyDifference: return readSamples(statType, sequenceIndex, bandIndex, includeAutos, includeFlags, true, progress); } } std::string TelescopeName() const { return _telescopeName; } void StoreFlags(const std::vector& flags, BaselineIntegrationDifferencing diffType, size_t sequenceIndex, size_t bandIndex, bool includeAutos); private: Result readSamples(BaselineIntegrationMode statType, size_t sequenceIndex, size_t bandIndex, bool includeAutos, bool includeFlags, bool freqDiff, ProgressListener& progress); Result readTimeDiff(BaselineIntegrationMode statType, size_t sequenceIndex, size_t bandIndex, bool includeAutos, bool includeFlags, ProgressListener& progress); void fillBand(AveragingMsReader::Result& result, bool freqDiff, const aocommon::MultiBandData& bands, size_t bandIndex); struct Statistic { double data1 = 0.0, data2 = 0.0, data3 = 0.0; size_t count = 0; void Add(BaselineIntegrationMode stat, std::complex sample) { ++count; switch (stat) { case BaselineIntegrationMode::Count: break; case BaselineIntegrationMode::Average: data1 += sample.real(); break; case BaselineIntegrationMode::AverageAbs: data1 += std::abs(sample); break; case BaselineIntegrationMode::Squared: data1 += std::norm(sample); break; case BaselineIntegrationMode::Stddev: data1 += sample.real(); data2 += sample.imag(); data3 += std::norm(sample); break; } } double Calculate(BaselineIntegrationMode stat) const { switch (stat) { case BaselineIntegrationMode::Count: return count; case BaselineIntegrationMode::Average: case BaselineIntegrationMode::AverageAbs: case BaselineIntegrationMode::Squared: return data1 / count; case BaselineIntegrationMode::Stddev: { double normMeanSq = std::norm(std::complex(data1, data2) / double(count)); return std::sqrt(data3 / count - normMeanSq); } } return 0.0; } }; Result makeResult(BaselineIntegrationMode statType, const Statistic* statsData, size_t nPolarizations, size_t nChannels, size_t nTimes); void storeFlags(const std::vector& flags, size_t sequenceIndex, size_t bandIndex, bool includeAutos); const std::string _filename; const std::string _dataColumn; size_t _nBands; std::vector _sequenceStart; std::string _telescopeName; }; #endif aoflagger-v3.5.1/msio/memorybaselinereader.cpp0000664000175000017500000002540414752462134017645 0ustar oleole#include "memorybaselinereader.h" #include "msselection.h" #include "../util/logger.h" #include "../util/progress/dummyprogresslistener.h" #include "../util/stopwatch.h" #include #include #include #include #include void MemoryBaselineReader::PrepareReadWrite(ProgressListener& progress) { if (!_isRead) { progress.OnStartTask("Reading measurement set into memory"); readSet(progress); _isRead = true; } } void MemoryBaselineReader::PerformReadRequests(ProgressListener& progress) { PrepareReadWrite(progress); for (size_t i = 0; i != _readRequests.size(); ++i) { const ReadRequest& request = _readRequests[i]; const BaselineID id(request.antenna1, request.antenna2, request.spectralWindow, request.sequenceId); const std::map>::const_iterator requestedBaselineIter = _baselines.find(id); if (requestedBaselineIter == _baselines.end()) { std::ostringstream errorStr; errorStr << "Exception in PerformReadRequests(): requested baseline is " "not available in measurement set " "(antenna1=" << request.antenna1 << ", antenna2=" << request.antenna2 << ", " "spw=" << request.spectralWindow << ", sequenceId=" << request.sequenceId << ")"; throw std::runtime_error(errorStr.str()); } else { _results.push_back(*requestedBaselineIter->second); } } _readRequests.clear(); progress.OnFinish(); } void MemoryBaselineReader::readSet(ProgressListener& progress) { const Stopwatch watch(true); initializeMeta(); const casacore::MeasurementSet table(OpenMS()); casacore::ScalarColumn ant1Column( table, casacore::MeasurementSet::columnName(casacore::MSMainEnums::ANTENNA1)), ant2Column(table, casacore::MeasurementSet::columnName( casacore::MSMainEnums::ANTENNA2)), dataDescIdColumn(table, casacore::MeasurementSet::columnName( casacore::MSMainEnums::DATA_DESC_ID)); casacore::ArrayColumn dataColumn(table, DataColumnName()); casacore::ArrayColumn flagColumn( table, casacore::MeasurementSet::columnName(casacore::MSMainEnums::FLAG)); casacore::ArrayColumn uvwColumn( table, casacore::MeasurementSet::columnName(casacore::MSMainEnums::UVW)); size_t antennaCount = MetaData().AntennaCount(), polarizationCount = Polarizations().size(), bandCount = MetaData().BandCount(), sequenceCount = MetaData().SequenceCount(), intStart = IntervalStart(), intEnd = IntervalEnd(); std::vector dataDescIdToSpw; MetaData().GetDataDescToBandVector(dataDescIdToSpw); std::vector bandInfos(bandCount); for (size_t b = 0; b != bandCount; ++b) bandInfos[b] = MetaData().GetBandInfo(b); // Initialize the look-up matrix // to quickly access the elements (without the map-lookup) typedef std::unique_ptr MatrixElement; typedef std::vector MatrixRow; typedef std::vector BaselineMatrix; typedef std::vector BaselineCube; BaselineCube baselineCube(sequenceCount * bandCount); for (size_t s = 0; s != sequenceCount; ++s) { for (size_t b = 0; b != bandCount; ++b) { BaselineMatrix& matrix = baselineCube[s * bandCount + b]; matrix.resize(antennaCount); for (size_t a1 = 0; a1 != antennaCount; ++a1) { matrix[a1].resize(antennaCount); for (size_t a2 = 0; a2 != antennaCount; ++a2) matrix[a1][a2] = nullptr; } } } // The actual reading of the data Logger::Debug << "Reading the data (interval={" << intStart << "..." << intEnd << "})...\n"; casacore::Array dataArray; casacore::Array flagArray; casacore::MeasurementSet ms(OpenMS()); MSSelection msSelection(ms, ObservationTimesPerSequence(), progress); msSelection.Process([&](size_t rowIndex, size_t sequenceId, size_t timeIndexInSequence) { size_t ant1 = ant1Column(rowIndex); size_t ant2 = ant2Column(rowIndex); const size_t spw = dataDescIdToSpw[dataDescIdColumn(rowIndex)]; const size_t spwFieldIndex = spw + sequenceId * bandCount; if (ant1 > ant2) std::swap(ant1, ant2); std::unique_ptr& result = baselineCube[spwFieldIndex][ant1][ant2]; if (result == nullptr) { const size_t timeStepCount = ObservationTimes(sequenceId).size(); const size_t nFreq = MetaData().FrequencyCount(spw); result.reset(new Result()); for (size_t p = 0; p != polarizationCount; ++p) { result->_realImages.emplace_back( Image2D::CreateZeroImagePtr(timeStepCount, nFreq)); result->_imaginaryImages.emplace_back( Image2D::CreateZeroImagePtr(timeStepCount, nFreq)); result->_flags.emplace_back( Mask2D::CreateSetMaskPtr(timeStepCount, nFreq)); } result->_bandInfo = bandInfos[spw]; result->_uvw.resize(timeStepCount); } dataArray = dataColumn.get(rowIndex); flagArray = flagColumn.get(rowIndex); casacore::Array uvwArray = uvwColumn.get(rowIndex); casacore::Array::const_contiter uvwPtr = uvwArray.cbegin(); UVW uvw; uvw.u = *uvwPtr; ++uvwPtr; uvw.v = *uvwPtr; ++uvwPtr; uvw.w = *uvwPtr; result->_uvw[timeIndexInSequence] = uvw; for (size_t p = 0; p != polarizationCount; ++p) { casacore::Array::const_contiter dataPtr = dataArray.cbegin(); casacore::Array::const_contiter flagPtr = flagArray.cbegin(); Image2D& real = *result->_realImages[p]; Image2D& imag = *result->_imaginaryImages[p]; Mask2D& mask = *result->_flags[p]; const size_t imgStride = real.Stride(); const size_t mskStride = mask.Stride(); num_t* realOutPtr = real.ValuePtr(timeIndexInSequence, 0); num_t* imagOutPtr = imag.ValuePtr(timeIndexInSequence, 0); bool* flagOutPtr = mask.ValuePtr(timeIndexInSequence, 0); for (size_t i = 0; i != p; ++i) { ++dataPtr; ++flagPtr; } const size_t frequencyCount = bandInfos[spw].channels.size(); for (size_t ch = 0; ch != frequencyCount; ++ch) { *realOutPtr = dataPtr->real(); *imagOutPtr = dataPtr->imag(); *flagOutPtr = *flagPtr; realOutPtr += imgStride; imagOutPtr += imgStride; flagOutPtr += mskStride; for (size_t i = 0; i != polarizationCount; ++i) { ++dataPtr; ++flagPtr; } } } }); // Move elements from matrix into the baseline map. for (size_t s = 0; s != sequenceCount; ++s) { for (size_t b = 0; b != bandCount; ++b) { const size_t fbIndex = s * bandCount + b; for (size_t a1 = 0; a1 != antennaCount; ++a1) { for (size_t a2 = a1; a2 != antennaCount; ++a2) { std::unique_ptr& result = baselineCube[fbIndex][a1][a2]; if (result) { _baselines.emplace(BaselineID(a1, a2, b, s), std::move(result)); } } } } } _areFlagsChanged = false; Logger::Debug << "Reading took " << watch.ToString() << ".\n"; } void MemoryBaselineReader::PerformFlagWriteRequests() { PrepareReadWrite(dummy_progress_); for (size_t i = 0; i != _writeRequests.size(); ++i) { const FlagWriteRequest& request = _writeRequests[i]; const BaselineID id(request.antenna1, request.antenna2, request.spectralWindow, request.sequenceId); std::unique_ptr& result = _baselines[id]; if (result->_flags.size() != request.flags.size()) throw std::runtime_error("Polarizations do not match"); for (size_t p = 0; p != result->_flags.size(); ++p) result->_flags[p].reset(new Mask2D(*request.flags[p])); } _areFlagsChanged = true; _writeRequests.clear(); } void MemoryBaselineReader::WriteToMs() { casacore::MeasurementSet ms(OpenMS(true)); casacore::ScalarColumn ant1Column( ms, casacore::MeasurementSet::columnName(casacore::MSMainEnums::ANTENNA1)), ant2Column(ms, casacore::MeasurementSet::columnName( casacore::MSMainEnums::ANTENNA2)), dataDescIdColumn(ms, casacore::MeasurementSet::columnName( casacore::MSMainEnums::DATA_DESC_ID)); casacore::ArrayColumn flagColumn( ms, casacore::MeasurementSet::columnName(casacore::MSMainEnums::FLAG)); std::vector dataIdToSpw; MetaData().GetDataDescToBandVector(dataIdToSpw); const size_t polarizationCount = Polarizations().size(); Logger::Debug << "Flags have changed, writing them back to the set...\n"; DummyProgressListener dummy; MSSelection msSelection(ms, ObservationTimesPerSequence(), dummy); msSelection.Process([&](size_t rowIndex, size_t sequenceId, size_t timeIndexInSequence) { size_t ant1 = ant1Column(rowIndex); size_t ant2 = ant2Column(rowIndex); const size_t spw = dataIdToSpw[dataDescIdColumn(rowIndex)]; if (ant1 > ant2) std::swap(ant1, ant2); const size_t frequencyCount = MetaData().FrequencyCount(spw); casacore::IPosition flagShape = casacore::IPosition(2); flagShape[0] = polarizationCount; flagShape[1] = frequencyCount; casacore::Array flagArray(flagShape); const BaselineID baselineID(ant1, ant2, spw, sequenceId); const std::map>::iterator resultIter = _baselines.find(baselineID); std::unique_ptr& result = resultIter->second; casacore::Array::contiter flagPtr = flagArray.cbegin(); std::vector masks(polarizationCount); for (size_t p = 0; p != polarizationCount; ++p) masks[p] = result->_flags[p].get(); for (size_t ch = 0; ch != frequencyCount; ++ch) { for (size_t p = 0; p != polarizationCount; ++p) { *flagPtr = masks[p]->Value(timeIndexInSequence, ch); ++flagPtr; } } flagColumn.put(rowIndex, flagArray); }); _areFlagsChanged = false; } bool MemoryBaselineReader::IsEnoughMemoryAvailable(uint64_t size) { const uint64_t totalMem = aocommon::system::TotalMemory(); if (size * 2 >= totalMem) { Logger::Warn << (size / 1000000) << " MB required, but " << (totalMem / 1000000) << " MB available.\n" "Because this is not at least twice as much, the reordering " "mode (slower!) will be used.\n"; return false; } else { Logger::Debug << (size / 1000000) << " MB required, " << (totalMem / 1000000) << " MB available: will use memory read mode.\n"; return true; } } aoflagger-v3.5.1/msio/spatialtimeloader.h0000664000175000017500000000163714752462134016621 0ustar oleole#ifndef SPATIALTIMELOADER_H #define SPATIALTIMELOADER_H #include #include #include "../structures/timefrequencydata.h" #include "../structures/msmetadata.h" /** * Loader for time x baseline matrices. These are mainly used for SVD * experiments. This class is used in the SpatialTimeImageSet . */ class SpatialTimeLoader { public: explicit SpatialTimeLoader(MSMetaData& measurementSet); ~SpatialTimeLoader(); TimeFrequencyData Load(unsigned channelIndex, bool fringeStop = true); unsigned ChannelCount() const { return _channelCount; } unsigned TimestepsCount() const { return _timestepsCount; } private: MSMetaData& _msMetaData; std::unique_ptr _sortedTable; std::unique_ptr _tableIter; unsigned _channelCount; unsigned _timestepsCount; unsigned _antennaCount; unsigned _polarizationCount; }; #endif aoflagger-v3.5.1/msio/baselinereader.h0000664000175000017500000002114714752462134016061 0ustar oleole#ifndef BASELINEREADER_H #define BASELINEREADER_H #include "../structures/antennainfo.h" #include "../structures/image2d.h" #include "../structures/mask2d.h" #include "../structures/msmetadata.h" #include #include #include #include #include #include #include typedef std::shared_ptr BaselineReaderPtr; typedef std::shared_ptr BaselineReaderCPtr; class BaselineReader { public: explicit BaselineReader(const std::string& msFile); virtual ~BaselineReader(); /** * Has the measurement set been modified? * * When it's been modified the changes need to be written to the measurement * set. By default the destructor of the subclasses should execute this * operation. In order to allow writing to multiple measurement sets in * parallel the functionality is exposed. */ virtual bool IsModified() const = 0; /** * Writes the changes to the measurement set. * * @post @c IsModified() == @c false. */ virtual void WriteToMs() = 0; /** * Prepares the reader before usage. * * Some readers have a preparation step that can be done in parallel. Calling * this function is optional; when not called manually the reader shall * execute the preparation itself. * * @note When no @a progress is needed use the @ref dummy_progress_. */ virtual void PrepareReadWrite(class ProgressListener& progress) = 0; static class DummyProgressListener dummy_progress_; bool ReadFlags() const { return _readFlags; } void SetReadFlags(bool readFlags) { _readFlags = readFlags; } bool ReadData() const { return _readData; } void SetReadData(bool readData) { _readData = readData; } const std::string& DataColumnName() const { return _dataColumnName; } void SetDataColumnName(const std::string& name) { _dataColumnName = name; } const std::vector& Polarizations() { initializePolarizations(); return _polarizations; } casacore::MeasurementSet OpenMS(bool writeAccess = false) const { if (writeAccess) return casacore::MeasurementSet(_msMetaData.Path(), casacore::TableLock::PermanentLockingWait, casacore::Table::Update); else return casacore::MeasurementSet( _msMetaData.Path(), casacore::TableLock::PermanentLockingWait); } MSMetaData& MetaData() { return _msMetaData; } const std::map& ObservationTimes(size_t sequenceId) const { return _observationTimes[sequenceId]; } std::vector ObservationTimes(size_t startIndex, size_t endIndex) const { std::vector times; times.insert(times.begin(), _observationTimesVector.begin() + startIndex, _observationTimesVector.begin() + endIndex); return times; } void AddReadRequest(size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId); void AddReadRequest(size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId, size_t startIndex, size_t endIndex) { addReadRequest(antenna1, antenna2, spectralWindow, sequenceId, startIndex, endIndex); } virtual void PerformReadRequests(class ProgressListener& progress) = 0; void AddWriteTask(std::vector flags, size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId) { initializePolarizations(); if (flags.size() != _polarizations.size()) { std::stringstream s; s << "Trying to write image with " << flags.size() << " polarizations to a measurement set with " << _polarizations.size(); throw std::runtime_error(s.str()); } FlagWriteRequest task; task.flags = flags; task.antenna1 = antenna1; task.antenna2 = antenna2; task.spectralWindow = spectralWindow; task.sequenceId = sequenceId; task.startIndex = 0; task.endIndex = flags[0]->Width(); task.leftBorder = 0; task.rightBorder = 0; _writeRequests.push_back(task); } virtual void PerformFlagWriteRequests() = 0; virtual void PerformDataWriteTask(std::vector _realImages, std::vector _imaginaryImages, size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId) = 0; TimeFrequencyData GetNextResult(std::vector& uvw); virtual size_t GetMinRecommendedBufferSize(size_t threadCount) { return threadCount; } virtual size_t GetMaxRecommendedBufferSize(size_t threadCount) { return 2 * threadCount; } static uint64_t MeasurementSetDataSize(const std::string& filename); /** * Returns an estimate of the size of the measurement set. * * This estimate can be used to see whether the memory reader can be used. * * The \a start and \a end are an optional, this allows using * \ref Options::startTimestep and \ref Options::endTimestep in this function * call. */ static uint64_t MeasurementSetIntervalDataSize(const string& filename, std::optional start, std::optional end); void SetInterval(std::optional start, std::optional end) { _intervalStart = start; _intervalEnd = end; if (_intervalStart) _msMetaData.SetIntervalStart(IntervalStart()); if (_intervalEnd) _msMetaData.SetIntervalEnd(IntervalEnd()); } bool HasIntervalStart() const { return (bool)_intervalStart; } bool HasIntervalEnd() const { return (bool)_intervalEnd; } size_t IntervalStart() const { if (HasIntervalStart()) return *_intervalStart; else return 0; } size_t IntervalEnd() const { if (HasIntervalEnd()) return *_intervalEnd; else return _observationTimesVector.size(); } protected: struct ReadRequest { int antenna1; int antenna2; int spectralWindow; unsigned sequenceId; size_t startIndex; size_t endIndex; }; struct FlagWriteRequest { FlagWriteRequest() = default; FlagWriteRequest(const FlagWriteRequest& source) : flags(source.flags), antenna1(source.antenna1), antenna2(source.antenna2), spectralWindow(source.spectralWindow), sequenceId(source.sequenceId), startIndex(source.startIndex), endIndex(source.endIndex), leftBorder(source.leftBorder), rightBorder(source.rightBorder) {} std::vector flags; int antenna1; int antenna2; int spectralWindow; unsigned sequenceId; size_t startIndex; size_t endIndex; size_t leftBorder; size_t rightBorder; }; struct Result { Result() = default; Result(const Result& source) : _realImages(source._realImages), _imaginaryImages(source._imaginaryImages), _flags(source._flags), _uvw(source._uvw), _bandInfo(source._bandInfo) {} std::vector _realImages; std::vector _imaginaryImages; std::vector _flags; std::vector _uvw; BandInfo _bandInfo; }; void initializeMeta() { initObservationTimes(); initializePolarizations(); } const std::vector>& ObservationTimesPerSequence() const { return _observationTimes; } std::vector _readRequests; std::vector _writeRequests; std::vector _results; private: BaselineReader(const BaselineReader&) = delete; BaselineReader& operator=(const BaselineReader&) = delete; void initializePolarizations(); void initObservationTimes(); void addReadRequest(size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId, size_t startIndex, size_t endIndex) { ReadRequest request; request.antenna1 = antenna1; request.antenna2 = antenna2; request.spectralWindow = spectralWindow; request.sequenceId = sequenceId; request.startIndex = startIndex; request.endIndex = endIndex; _readRequests.push_back(request); } MSMetaData _msMetaData; std::string _dataColumnName; bool _readData, _readFlags; std::vector> _observationTimes; std::vector _observationTimesVector; std::vector _polarizations; std::optional _intervalStart, _intervalEnd; }; #endif // BASELINEREADER_H aoflagger-v3.5.1/msio/rspreader.h0000664000175000017500000001347514752462134015110 0ustar oleole#ifndef RSPREADER_H #define RSPREADER_H #include #include "../util/logger.h" #include "../structures/timefrequencydata.h" #include "../structures/timefrequencymetadata.h" class RSPReader { public: explicit RSPReader(const std::string& rawFile) : _rawFile(rawFile), _clockSpeed(200000000) {} ~RSPReader() {} std::pair ReadChannelBeamlet( unsigned long timestepStart, unsigned long timestepEnd, unsigned beamletCount, unsigned beamletIndex); std::pair ReadSingleBeamlet( unsigned long timestepStart, unsigned long timestepEnd, unsigned beamletCount, unsigned beamletIndex); std::pair ReadAllBeamlets( unsigned long timestepStart, unsigned long timestepEnd, unsigned beamletCount); void ReadForStatistics(unsigned beamletCount); const std::string& File() const { return _rawFile; } unsigned long TimeStepCount(size_t beamletCount) const; private: static const unsigned char BitReverseTable256[256]; static signed short toShort(unsigned char c1, unsigned char c2) { return (c2 << 8) | c1; } static unsigned short toUShort(unsigned char c1, unsigned char c2) { return (c1 << 8) | c2; } static unsigned int toUInt(unsigned char c1, unsigned char c2, unsigned char c3, unsigned char c4) { return (c1 << 24) | (c2 << 16) | (c3 << 8) | c4; } const std::string _rawFile; const unsigned long _clockSpeed; static const unsigned long STATION_INTEGRATION_STEPS; struct RCPTransportHeader { unsigned char versionAndHeaderLength : 8; unsigned char typeOfService : 8; unsigned short totalLength : 16; unsigned short identification : 16; unsigned int flagsAndFragmentOffset : 8; unsigned int ttl : 8; unsigned int protocol : 8; unsigned int headerCrc : 16; unsigned int sourceIP : 32; unsigned int destinationIP : 32; unsigned int udpSourcePort : 16; unsigned int udpDestPort : 16; unsigned int udpLength : 16; unsigned int udpChecksum : 16; void Print() const { Logger::Debug << "versionAndHeaderLength=" << versionAndHeaderLength << "\n" << "typeOfService=" << typeOfService << "\n" << "totalLength=" << totalLength << "\n" << "identification=" << identification << "\n" << "flagsAndFragmentOffset=" << flagsAndFragmentOffset << "\n" << "ttl=" << ttl << "\n" << "protocol=" << protocol << "\n" << "headerCrc=" << headerCrc << "\n" << "sourceIP=" << sourceIP << "\n" << "destinationIP=" << destinationIP << "\n" << "udpSourcePort=" << udpSourcePort << "\n" << "udpDestPort=" << udpDestPort << "\n" << "udpLength=" << udpLength << "\n" << "udpChecksum=" << udpChecksum << "\n"; } }; struct RCPApplicationHeader { unsigned char versionId; unsigned char sourceInfo; unsigned short configurationId; unsigned short stationId; unsigned char nofBeamlets; unsigned char nofBlocks; unsigned int timestamp; unsigned int blockSequenceNumber; static const unsigned int SIZE; void Read(std::ifstream& stream) { unsigned char buffer[16]; stream.read(reinterpret_cast(buffer), 16); versionId = buffer[0]; sourceInfo = buffer[1]; configurationId = toUShort(buffer[3], buffer[2]); stationId = toUShort(buffer[5], buffer[4]); nofBeamlets = buffer[6]; nofBlocks = buffer[7]; timestamp = toUInt(buffer[11], buffer[10], buffer[9], buffer[8]); blockSequenceNumber = toUInt(buffer[15], buffer[14], buffer[13], buffer[12]); } void Print() const { Logger::Debug << "versionId=" << (unsigned int)versionId << "\n" << "sourceInfo=" << (unsigned int)sourceInfo << "\n" << "configurationId=" << (unsigned int)configurationId << "\n" << "stationId=" << (unsigned int)stationId << "\n" << "nofBeamlets=" << (unsigned int)nofBeamlets << "\n" << "nofBlocks=" << (unsigned int)nofBlocks << "\n" << "timestamp=" << (unsigned int)timestamp << "\n" << "blockSequenceNumber=" << (unsigned int)blockSequenceNumber << "\n"; } }; struct RCPBeamletData { signed short xr, xi; signed short yr, yi; static const unsigned int SIZE; void Read(std::ifstream& stream) { unsigned char buffer[8]; stream.read(reinterpret_cast(buffer), 8); xr = toShort(buffer[6], buffer[7]); xi = toShort(buffer[4], buffer[5]); yr = toShort(buffer[2], buffer[3]); yi = toShort(buffer[0], buffer[1]); } void Print() { Logger::Debug << "x=" << xr; if (xi > 0) Logger::Debug << "+" << xi << "i"; else Logger::Debug << "-" << (-xi) << "i"; Logger::Debug << ",y=" << yr; if (yi > 0) Logger::Debug << "+" << yi << "i\n"; else Logger::Debug << "-" << (-yi) << "i\n"; } }; struct BeamletStatistics { BeamletStatistics() : totalCount(0) { for (unsigned i = 0; i < 16; ++i) bitUseCount[i] = 0; } void Print() { for (unsigned bit = 0; bit < 16; ++bit) { Logger::Info << "Bit " << bit << " times required: " << bitUseCount[bit] << " (" << (100.0 * (double)bitUseCount[bit] / (double)totalCount) << "%)" << '\n'; } } unsigned long totalCount; unsigned long bitUseCount[16]; }; }; #endif // RSPREADER_H aoflagger-v3.5.1/msio/singlebaselinefile.h0000664000175000017500000000237114752462134016736 0ustar oleole#ifndef SINGLE_BASELINE_FILE_H #define SINGLE_BASELINE_FILE_H #include #include #include "../structures/timefrequencydata.h" #include "../structures/timefrequencymetadata.h" #include "../util/progress/progresslistener.h" class SingleBaselineFile { public: void Read(std::istream& stream, ProgressListener& progress); void Write(std::ostream& stream); static void Serialize(std::ostream& stream, const TimeFrequencyData& data); static void Serialize(std::ostream& stream, const TimeFrequencyMetaData& metaData); static void Serialize(std::ostream& stream, const Image2D& image); static void Serialize(std::ostream& stream, const Mask2D& mask); static TimeFrequencyData UnserializeTFData(std::istream& stream, ProgressListener& progress); static TimeFrequencyMetaData UnserializeMetaData(std::istream& stream); static Image2D UnserializeImage(std::istream& stream, ProgressListener& progress, size_t progressOffset, size_t progressMax); static Mask2D UnserializeMask(std::istream& stream); TimeFrequencyData data; TimeFrequencyMetaData metaData; std::string telescopeName; }; #endif aoflagger-v3.5.1/msio/baselinematrixloader.cpp0000664000175000017500000003123314752462134017642 0ustar oleole#include "baselinematrixloader.h" #include #include #include #include "../structures/spatialmatrixmetadata.h" BaselineMatrixLoader::BaselineMatrixLoader(MSMetaData& msMetaData) : _sortedTable(), _tableIter(), _currentIterIndex(0), _msMetaData(msMetaData), _timeIndexCount(0), _metaData() { const casacore::Table rawTable(_msMetaData.Path()); casacore::Block names(4); names[0] = "DATA_DESC_ID"; names[1] = "TIME"; names[2] = "ANTENNA1"; names[3] = "ANTENNA2"; _sortedTable.reset(new casacore::Table(rawTable.sort(names))); casacore::Block selectionNames(2); selectionNames[0] = "DATA_DESC_ID"; selectionNames[1] = "TIME"; casacore::TableIterator iter(*_sortedTable, selectionNames, casacore::TableIterator::Ascending, casacore::TableIterator::NoSort); while (!iter.pastEnd()) { iter.next(); ++_timeIndexCount; } _frequencyCount = _msMetaData.FrequencyCount(0); _tableIter.reset(new casacore::TableIterator( *_sortedTable, selectionNames, casacore::TableIterator::Ascending, casacore::TableIterator::NoSort)); } TimeFrequencyData BaselineMatrixLoader::LoadSummed(size_t timeIndex) { casacore::Block selectionNames(2); selectionNames[0] = "DATA_DESC_ID"; selectionNames[1] = "TIME"; if (timeIndex < _currentIterIndex) { _tableIter.reset(new casacore::TableIterator( *_sortedTable, selectionNames, casacore::TableIterator::Ascending, casacore::TableIterator::NoSort)); _currentIterIndex = 0; } while (!_tableIter->pastEnd() && timeIndex - _currentIterIndex > 0) { _tableIter->next(); ++_currentIterIndex; } if (_tableIter->pastEnd()) { throw std::runtime_error("Time index not found"); } const casacore::Table table = _tableIter->table(); const casacore::ScalarColumn antenna1Column(table, "ANTENNA1"); const casacore::ScalarColumn antenna2Column(table, "ANTENNA2"); // Find highest antenna index int nrAntenna = 0; for (size_t i = 0; i < table.nrow(); ++i) { int a1 = antenna1Column(i), a2 = antenna2Column(i); if (a1 > nrAntenna) nrAntenna = a1; if (a2 > nrAntenna) nrAntenna = a2; } ++nrAntenna; _metaData.reset(new SpatialMatrixMetaData(nrAntenna)); const casacore::ArrayColumn flagColumn(table, "FLAG"); const casacore::ArrayColumn dataColumn(table, "DATA"); const casacore::ArrayColumn uvwColumn(table, "UVW"); Image2DPtr xxRImage = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), xxIImage = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), xyRImage = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), xyIImage = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), yxRImage = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), yxIImage = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), yyRImage = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), yyIImage = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna); Mask2DPtr xxMask = Mask2D::CreateUnsetMaskPtr(nrAntenna, nrAntenna), xyMask = Mask2D::CreateUnsetMaskPtr(nrAntenna, nrAntenna), yxMask = Mask2D::CreateUnsetMaskPtr(nrAntenna, nrAntenna), yyMask = Mask2D::CreateUnsetMaskPtr(nrAntenna, nrAntenna); for (size_t j = 0; j < table.nrow(); ++j) { int a1 = antenna1Column(j), a2 = antenna2Column(j); casacore::Array data = dataColumn(j); casacore::Array flags = flagColumn(j); casacore::Array::const_iterator i = data.begin(); casacore::Array::const_iterator fI = flags.begin(); num_t xxr = 0.0, xxi = 0.0, xyr = 0.0, xyi = 0.0, yxr = 0.0, yxi = 0.0, yyr = 0.0, yyi = 0.0; size_t xxc = 0, xyc = 0, yxc = 0, yyc = 0; for (size_t f = 0; f < (size_t)_frequencyCount; ++f) { const casacore::Complex& xx = *i; ++i; const casacore::Complex& xy = *i; ++i; const casacore::Complex& yx = *i; ++i; const casacore::Complex& yy = *i; ++i; const bool xxF = *fI; ++fI; const bool xyF = *fI; ++fI; const bool yxF = *fI; ++fI; const bool yyF = *fI; ++fI; if (std::isfinite(xxr) && std::isfinite(xxi) && !xxF) { xxr += xx.real(); xxi += xx.imag(); ++xxc; } if (std::isfinite(xyr) && std::isfinite(xyi) && !xyF) { xyr += xy.real(); xyi += xy.imag(); ++xyc; } if (std::isfinite(yxr) && std::isfinite(yxi) && !yxF) { yxr += yx.real(); yxi += yx.imag(); ++yxc; } if (std::isfinite(yyr) && std::isfinite(yyi) && !yyF) { yyr += yy.real(); yyi += yy.imag(); ++yyc; } } xxRImage->SetValue(a1, a2, xxr / xxc); xxIImage->SetValue(a1, a2, xxi / xxc); xyRImage->SetValue(a1, a2, xyr / xxc); xyIImage->SetValue(a1, a2, xyi / xxc); yxRImage->SetValue(a1, a2, yxr / xxc); yxIImage->SetValue(a1, a2, yxi / xxc); yyRImage->SetValue(a1, a2, yyr / xxc); yyIImage->SetValue(a1, a2, yyi / xxc); xxMask->SetValue(a1, a2, xxc == 0); xyMask->SetValue(a1, a2, xyc == 0); yxMask->SetValue(a1, a2, yxc == 0); yyMask->SetValue(a1, a2, yyc == 0); UVW uvw; casacore::Array arr = uvwColumn(j); casacore::Array::const_iterator uvwArrayIterator = arr.begin(); uvw.u = *uvwArrayIterator; ++uvwArrayIterator; uvw.v = *uvwArrayIterator; ++uvwArrayIterator; uvw.w = *uvwArrayIterator; _metaData->SetUVW(a1, a2, uvw); if (a1 != a2) { xxRImage->SetValue(a2, a1, xxr / xxc); xxIImage->SetValue(a2, a1, -xxi / xxc); xyRImage->SetValue(a2, a1, xyr / xxc); xyIImage->SetValue(a2, a1, -xyi / xxc); yxRImage->SetValue(a2, a1, yxr / xxc); yxIImage->SetValue(a2, a1, -yxi / xxc); yyRImage->SetValue(a2, a1, yyr / xxc); yyIImage->SetValue(a2, a1, -yyi / xxc); xxMask->SetValue(a2, a1, xxc == 0); xyMask->SetValue(a2, a1, xyc == 0); yxMask->SetValue(a2, a1, yxc == 0); yyMask->SetValue(a2, a1, yyc == 0); uvw.u = -uvw.u; uvw.v = -uvw.v; uvw.w = -uvw.w; _metaData->SetUVW(a2, a1, uvw); } } const casacore::ScalarColumn bandColumn(table, "DATA_DESC_ID"); const BandInfo band = _msMetaData.GetBandInfo(bandColumn(0)); _metaData->SetFrequency(band.CenterFrequencyHz()); TimeFrequencyData data = TimeFrequencyData::FromLinear(xxRImage, xxIImage, xyRImage, xyIImage, yxRImage, yxIImage, yyRImage, yyIImage); data.SetIndividualPolarizationMasks(xxMask, xyMask, yxMask, yyMask); return data; } void BaselineMatrixLoader::LoadPerChannel( size_t timeIndex, std::vector& data) { casacore::Block selectionNames(2); selectionNames[0] = "DATA_DESC_ID"; selectionNames[1] = "TIME"; if (timeIndex < _currentIterIndex) { _tableIter.reset(new casacore::TableIterator( *_sortedTable, selectionNames, casacore::TableIterator::Ascending, casacore::TableIterator::NoSort)); _currentIterIndex = 0; } while (!_tableIter->pastEnd() && timeIndex - _currentIterIndex > 0) { _tableIter->next(); ++_currentIterIndex; } if (_tableIter->pastEnd()) { throw std::runtime_error("Time index not found"); } const casacore::Table table = _tableIter->table(); const casacore::ROScalarColumn antenna1Column(table, "ANTENNA1"); const casacore::ROScalarColumn antenna2Column(table, "ANTENNA2"); // Find highest antenna index int nrAntenna = 0; for (size_t i = 0; i < table.nrow(); ++i) { int a1 = antenna1Column(i), a2 = antenna2Column(i); if (a1 > nrAntenna) nrAntenna = a1; if (a2 > nrAntenna) nrAntenna = a2; } ++nrAntenna; _metaData.reset(new SpatialMatrixMetaData(nrAntenna)); const casacore::ArrayColumn flagColumn(table, "FLAG"); const casacore::ArrayColumn dataColumn(table, "DATA"); const casacore::ArrayColumn uvwColumn(table, "UVW"); std::vector xxRImage(_frequencyCount), xxIImage(_frequencyCount), xyRImage(_frequencyCount), xyIImage(_frequencyCount), yxRImage(_frequencyCount), yxIImage(_frequencyCount), yyRImage(_frequencyCount), yyIImage(_frequencyCount); std::vector xxMask(_frequencyCount), xyMask(_frequencyCount), yxMask(_frequencyCount), yyMask(_frequencyCount); for (size_t f = 0; f < (size_t)_frequencyCount; ++f) { xxRImage[f] = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), xxIImage[f] = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), xyRImage[f] = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), xyIImage[f] = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), yxRImage[f] = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), yxIImage[f] = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), yyRImage[f] = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna), yyIImage[f] = Image2D::CreateZeroImagePtr(nrAntenna, nrAntenna); xxMask[f] = Mask2D::CreateSetMaskPtr(nrAntenna, nrAntenna), xyMask[f] = Mask2D::CreateSetMaskPtr(nrAntenna, nrAntenna), yxMask[f] = Mask2D::CreateSetMaskPtr(nrAntenna, nrAntenna), yyMask[f] = Mask2D::CreateSetMaskPtr(nrAntenna, nrAntenna); } for (size_t row = 0; row < table.nrow(); ++row) { int a1 = antenna1Column(row), a2 = antenna2Column(row); casacore::Array data = dataColumn(row); casacore::Array flags = flagColumn(row); casacore::Array::const_iterator i = data.begin(); casacore::Array::const_iterator fI = flags.begin(); for (size_t f = 0; f < (size_t)_frequencyCount; ++f) { const casacore::Complex xx = *i; ++i; const casacore::Complex xy = *i; ++i; const casacore::Complex yx = *i; ++i; const casacore::Complex yy = *i; ++i; bool xxF = *fI; ++fI; bool xyF = *fI; ++fI; bool yxF = *fI; ++fI; bool yyF = *fI; ++fI; if (!std::isfinite(xx.real()) || !std::isfinite(xx.imag())) xxF = true; if (!std::isfinite(xy.real()) || !std::isfinite(xy.imag())) xyF = true; if (!std::isfinite(yx.real()) || !std::isfinite(yx.imag())) yxF = true; if (!std::isfinite(yy.real()) || !std::isfinite(yy.imag())) yyF = true; xxRImage[f]->SetValue(a1, a2, xx.real()); xxIImage[f]->SetValue(a1, a2, xx.imag()); xyRImage[f]->SetValue(a1, a2, xy.real()); xyIImage[f]->SetValue(a1, a2, xy.imag()); yxRImage[f]->SetValue(a1, a2, yx.real()); yxIImage[f]->SetValue(a1, a2, yx.imag()); yyRImage[f]->SetValue(a1, a2, yy.real()); yyIImage[f]->SetValue(a1, a2, yy.imag()); xxMask[f]->SetValue(a1, a2, !xxF); xyMask[f]->SetValue(a1, a2, !xyF); yxMask[f]->SetValue(a1, a2, !yxF); yyMask[f]->SetValue(a1, a2, !yyF); if (a1 != a2) { xxRImage[f]->SetValue(a2, a1, xx.real()); xxIImage[f]->SetValue(a2, a1, -xx.imag()); xyRImage[f]->SetValue(a2, a1, xy.real()); xyIImage[f]->SetValue(a2, a1, -xy.imag()); yxRImage[f]->SetValue(a2, a1, yx.real()); yxIImage[f]->SetValue(a2, a1, -yx.imag()); yyRImage[f]->SetValue(a2, a1, yy.real()); yyIImage[f]->SetValue(a2, a1, -yy.imag()); xxMask[f]->SetValue(a2, a1, !xxF); xyMask[f]->SetValue(a2, a1, !xyF); yxMask[f]->SetValue(a2, a1, !yxF); yyMask[f]->SetValue(a2, a1, !yyF); } } UVW uvw; casacore::Array arr = uvwColumn(row); casacore::Array::const_iterator uvwArrayIterator = arr.begin(); uvw.u = *uvwArrayIterator; ++uvwArrayIterator; uvw.v = *uvwArrayIterator; ++uvwArrayIterator; uvw.w = *uvwArrayIterator; _metaData->SetUVW(a1, a2, uvw); if (a1 != a2) { uvw.u = -uvw.u; uvw.v = -uvw.v; uvw.w = -uvw.w; _metaData->SetUVW(a2, a1, uvw); } } const casacore::ScalarColumn bandColumn(table, "DATA_DESC_ID"); const BandInfo band = _msMetaData.GetBandInfo(bandColumn(0)); _metaData->SetFrequency(band.CenterFrequencyHz()); data.clear(); for (size_t f = 0; f < _frequencyCount; ++f) { TimeFrequencyData singleMatrix = TimeFrequencyData::FromLinear( xxRImage[f], xxIImage[f], xyRImage[f], xyIImage[f], yxRImage[f], yxIImage[f], yyRImage[f], yyIImage[f]); singleMatrix.SetIndividualPolarizationMasks(xxMask[f], xyMask[f], yxMask[f], yyMask[f]); data.push_back(singleMatrix); } } aoflagger-v3.5.1/msio/directbaselinereader.h0000664000175000017500000000666314752462134017262 0ustar oleole#ifndef DIRECTBASELINEREADER_H #define DIRECTBASELINEREADER_H #include #include #include #include "baselinereader.h" #include "../structures/antennainfo.h" #include "../structures/image2d.h" #include "../structures/mask2d.h" #include "../structures/msmetadata.h" class DirectBaselineReader final : public BaselineReader { public: explicit DirectBaselineReader(const std::string& msFile); bool IsModified() const override { return false; } void WriteToMs() override {} void PrepareReadWrite(ProgressListener&) override {} void PerformReadRequests(class ProgressListener& listener) override; void PerformFlagWriteRequests() override; void PerformDataWriteTask( [[maybe_unused]] std::vector realImages, [[maybe_unused]] std::vector imaginaryImages, [[maybe_unused]] size_t antenna1, [[maybe_unused]] size_t antenna2, [[maybe_unused]] size_t spectralWindow, [[maybe_unused]] size_t sequenceId) override { throw std::runtime_error( "The direct baseline reader can not write data back to file: use the " "indirect reader"); } std::vector ReadUVW(unsigned antenna1, unsigned antenna2, unsigned spectralWindow, unsigned sequenceId); private: class BaselineCacheIndex { public: BaselineCacheIndex() {} BaselineCacheIndex(const BaselineCacheIndex& source) : antenna1(source.antenna1), antenna2(source.antenna2), spectralWindow(source.spectralWindow), sequenceId(source.sequenceId) {} bool operator==(const BaselineCacheIndex& rhs) const { return antenna1 == rhs.antenna1 && antenna2 == rhs.antenna2 && spectralWindow == rhs.spectralWindow && sequenceId == rhs.sequenceId; } bool operator<(const BaselineCacheIndex& rhs) const { if (antenna1 < rhs.antenna1) { return true; } else if (antenna1 == rhs.antenna1) { if (antenna2 < rhs.antenna2) { return true; } else if (antenna2 == rhs.antenna2) { if (spectralWindow < rhs.spectralWindow) return true; else if (spectralWindow == rhs.spectralWindow) return sequenceId < rhs.sequenceId; } } return false; } size_t antenna1, antenna2, spectralWindow, sequenceId; }; struct BaselineCacheValue { std::vector rows; }; void initBaselineCache(); void addRequestRows(ReadRequest request, size_t requestIndex, std::vector>& rows); void addRequestRows(FlagWriteRequest request, size_t requestIndex, std::vector>& rows); void addRowToBaselineCache(size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId, size_t row); void readUVWData(); void readTimeData(size_t requestIndex, size_t xOffset, size_t frequencyCount, const casacore::Array& data); void readTimeFlags(size_t requestIndex, size_t xOffset, size_t frequencyCount, const casacore::Array& flag); void readWeights(size_t requestIndex, size_t xOffset, size_t frequencyCount, const casacore::Array& weight); std::map _baselineCache; casacore::MeasurementSet _ms; }; #endif // DIRECTBASELINEREADER_H aoflagger-v3.5.1/msio/singlebaselinefile.cpp0000664000175000017500000002146014752462134017271 0ustar oleole#include "singlebaselinefile.h" #include "../util/serializable.h" #include #define FILE_FORMAT_VERSION 1 void SingleBaselineFile::Read(std::istream& stream, ProgressListener& progress) { if (!stream) throw std::runtime_error("Could not open file"); char magic[9]; stream.read(magic, 8); magic[8] = 0; if (std::string(magic) != "RFIBL") throw std::runtime_error("This is not an AOFlagger single baseline file"); const unsigned fileformat = Serializable::UnserializeUInt32(stream); if (fileformat != FILE_FORMAT_VERSION) throw std::runtime_error( "This AOFlagger single baseline file has an unknown file format"); const std::string versionStr = Serializable::UnserializeString(stream); Serializable::UnserializeUInt32(stream); // maj Serializable::UnserializeUInt32(stream); // min Serializable::UnserializeUInt32(stream); // submin data = UnserializeTFData(stream, progress); metaData = UnserializeMetaData(stream); telescopeName = Serializable::UnserializeString(stream); } void SingleBaselineFile::Write(std::ostream& stream) { stream.write("RFIBL\0\0\0", 8); Serializable::SerializeToUInt32( stream, FILE_FORMAT_VERSION); // fileformat version index Serializable::SerializeToString(stream, AOFLAGGER_VERSION_DATE_STR); Serializable::SerializeToUInt32(stream, AOFLAGGER_VERSION_MAJOR); Serializable::SerializeToUInt32(stream, AOFLAGGER_VERSION_MINOR); Serializable::SerializeToUInt32(stream, AOFLAGGER_VERSION_SUBMINOR); Serialize(stream, data); Serialize(stream, metaData); Serializable::SerializeToString(stream, telescopeName); } TimeFrequencyData SingleBaselineFile::UnserializeTFData( std::istream& stream, ProgressListener& progress) { TimeFrequencyData data; const size_t polCount = Serializable::UnserializeUInt32(stream); const size_t complCode = Serializable::UnserializeUInt32(stream); enum TimeFrequencyData::ComplexRepresentation repr; switch (complCode) { default: case 0: repr = TimeFrequencyData::PhasePart; break; case 1: repr = TimeFrequencyData::AmplitudePart; break; case 2: repr = TimeFrequencyData::RealPart; break; case 3: repr = TimeFrequencyData::ImaginaryPart; break; case 4: repr = TimeFrequencyData::ComplexParts; break; } for (size_t i = 0; i != polCount; ++i) { TimeFrequencyData polData; const size_t polCode = Serializable::UnserializeUInt32(stream); const aocommon::PolarizationEnum pol = aocommon::Polarization::AipsIndexToEnum(polCode); const uint32_t imageFlagBitset = Serializable::UnserializeUInt32(stream); const size_t imageCount = imageFlagBitset & 0x03; const size_t maskCount = (imageFlagBitset & 0x04) ? 1 : 0; if (imageCount == 2) { Image2D first = UnserializeImage(stream, progress, i * 2, polCount * 2), second = UnserializeImage(stream, progress, i * 2 + 1, polCount * 2); polData = TimeFrequencyData(pol, Image2D::MakePtr(first), Image2D::MakePtr(second)); } else if (imageCount == 1) { polData = TimeFrequencyData( repr, pol, Image2D::MakePtr(UnserializeImage(stream, progress, i, polCount))); } if (maskCount == 1) polData.SetGlobalMask(Mask2D::MakePtr(UnserializeMask(stream))); if (i == 0) data = polData; else data = TimeFrequencyData::MakeFromPolarizationCombination(data, polData); } return data; } TimeFrequencyMetaData SingleBaselineFile::UnserializeMetaData( std::istream& stream) { TimeFrequencyMetaData metaData; const size_t featureSet = Serializable::UnserializeUInt64(stream); const bool hasAntenna1 = featureSet & 0x01; const bool hasAntenna2 = featureSet & 0x02; const bool hasBand = featureSet & 0x04; const bool hasObsTimes = featureSet & 0x10; if (hasAntenna1) { AntennaInfo ant; ant.Unserialize(stream); metaData.SetAntenna1(ant); } if (hasAntenna2) { AntennaInfo ant; ant.Unserialize(stream); metaData.SetAntenna2(ant); } if (hasBand) { BandInfo band; band.Unserialize(stream); metaData.SetBand(band); } if (hasObsTimes) { std::vector vals(Serializable::UnserializeUInt64(stream)); for (double& t : vals) t = Serializable::UnserializeDouble(stream); metaData.SetObservationTimes(vals); } return metaData; } Image2D SingleBaselineFile::UnserializeImage(std::istream& stream, ProgressListener& progress, size_t progressOffset, size_t progressMax) { size_t width = Serializable::UnserializeUInt64(stream), height = Serializable::UnserializeUInt64(stream); Image2D result = Image2D::MakeUnsetImage(width, height); for (size_t y = 0; y != height; ++y) { progress.OnProgress(height * progressOffset + y, height * progressMax); for (size_t x = 0; x != width; ++x) { result.SetValue(x, y, Serializable::UnserializeFloat(stream)); } } return result; } Mask2D SingleBaselineFile::UnserializeMask(std::istream& stream) { size_t width = Serializable::UnserializeUInt64(stream), height = Serializable::UnserializeUInt64(stream); Mask2D result = Mask2D::MakeUnsetMask(width, height); for (size_t y = 0; y != height; ++y) { for (size_t x = 0; x != width; ++x) { char val; stream.read(&val, 1); result.SetValue(x, y, val != 0); } } return result; } void SingleBaselineFile::Serialize(std::ostream& stream, const TimeFrequencyData& data) { Serializable::SerializeToUInt32(stream, data.PolarizationCount()); int complCode; switch (data.ComplexRepresentation()) { default: case TimeFrequencyData::PhasePart: complCode = 0; break; case TimeFrequencyData::AmplitudePart: complCode = 1; break; case TimeFrequencyData::RealPart: complCode = 2; break; case TimeFrequencyData::ImaginaryPart: complCode = 3; break; case TimeFrequencyData::ComplexParts: complCode = 4; break; } Serializable::SerializeToUInt32(stream, complCode); for (size_t i = 0; i != data.PolarizationCount(); ++i) { const aocommon::PolarizationEnum p = data.GetPolarization(i); Serializable::SerializeToUInt32(stream, aocommon::Polarization::EnumToAipsIndex(p)); unsigned int imageFlagBitset = 0; const TimeFrequencyData polData = data.MakeFromPolarizationIndex(i); if (polData.ImageCount() == 2) imageFlagBitset = imageFlagBitset | 0x02; if (polData.ImageCount() == 1) imageFlagBitset = imageFlagBitset | 0x01; if (polData.MaskCount() == 1) imageFlagBitset = imageFlagBitset | 0x04; Serializable::SerializeToUInt32(stream, imageFlagBitset); if (polData.ImageCount() >= 1) Serialize(stream, *polData.GetImage(0)); if (polData.ImageCount() == 2) Serialize(stream, *polData.GetImage(1)); if (polData.MaskCount() == 1) Serialize(stream, *polData.GetMask(0)); } } void SingleBaselineFile::Serialize(std::ostream& stream, const TimeFrequencyMetaData& metaData) { size_t featureSet = 0; if (metaData.HasAntenna1()) featureSet = featureSet | 0x01; if (metaData.HasAntenna2()) featureSet = featureSet | 0x02; if (metaData.HasBand()) featureSet = featureSet | 0x04; // if(metaData.HasField()) // featureSet = featureSet | 0x08; if (metaData.HasObservationTimes()) featureSet = featureSet | 0x10; // if(metaData.HasUVW()) // featureSet = featureSet | 0x20; Serializable::SerializeToUInt64(stream, featureSet); if (metaData.HasAntenna1()) metaData.Antenna1().Serialize(stream); if (metaData.HasAntenna2()) metaData.Antenna2().Serialize(stream); if (metaData.HasBand()) metaData.Band().Serialize(stream); if (metaData.HasObservationTimes()) { const std::vector& vals = metaData.ObservationTimes(); Serializable::SerializeToUInt64(stream, vals.size()); for (const double& t : vals) Serializable::SerializeToDouble(stream, t); } } void SingleBaselineFile::Serialize(std::ostream& stream, const Image2D& image) { Serializable::SerializeToUInt64(stream, image.Width()); Serializable::SerializeToUInt64(stream, image.Height()); for (size_t y = 0; y != image.Height(); ++y) { for (size_t x = 0; x != image.Width(); ++x) { Serializable::SerializeToFloat(stream, image.Value(x, y)); } } } void SingleBaselineFile::Serialize(std::ostream& stream, const Mask2D& mask) { Serializable::SerializeToUInt64(stream, mask.Width()); Serializable::SerializeToUInt64(stream, mask.Height()); for (size_t y = 0; y != mask.Height(); ++y) { for (size_t x = 0; x != mask.Width(); ++x) { const char val = mask.Value(x, y) ? 1 : 0; stream.write(&val, 1); } } } aoflagger-v3.5.1/msio/baselinereader.cpp0000664000175000017500000001630714752462134016416 0ustar oleole#include "baselinereader.h" #include #include #include #include #include #include #include #include #include #include #include "../util/progress/dummyprogresslistener.h" #include "../structures/timefrequencydata.h" #include "../util/logger.h" DummyProgressListener BaselineReader::dummy_progress_; BaselineReader::BaselineReader(const std::string& msFile) : _msMetaData(msFile), _dataColumnName("DATA"), _readData(true), _readFlags(true), _polarizations() {} BaselineReader::~BaselineReader() {} void BaselineReader::initObservationTimes() { if (_observationTimes.empty()) { Logger::Debug << "Initializing observation times...\n"; const size_t sequenceCount = _msMetaData.SequenceCount(); _observationTimes.resize(sequenceCount); for (size_t sequenceId = 0; sequenceId != sequenceCount; ++sequenceId) { const std::set& times = _msMetaData.GetObservationTimesSet(sequenceId); unsigned index = 0; for (const double t : times) { _observationTimes[sequenceId].emplace(t, index); _observationTimesVector.push_back(t); ++index; } } } } void BaselineReader::AddReadRequest(size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId) { initObservationTimes(); addReadRequest(antenna1, antenna2, spectralWindow, sequenceId, 0, _observationTimes[sequenceId].size()); } TimeFrequencyData BaselineReader::GetNextResult(std::vector& uvw) { const size_t requestIndex = 0; TimeFrequencyData data; data = TimeFrequencyData(_polarizations.data(), _polarizations.size(), _results[requestIndex]._realImages.data(), _results[requestIndex]._imaginaryImages.data()); data.SetIndividualPolarizationMasks(_results[requestIndex]._flags.data()); uvw = _results[0]._uvw; _results.erase(_results.begin() + requestIndex); return data; } void BaselineReader::initializePolarizations() { if (_polarizations.empty()) { casacore::MeasurementSet ms(_msMetaData.Path()); const casacore::MSDataDescription ddTable = ms.dataDescription(); if (ddTable.nrow() == 0) throw std::runtime_error("DataDescription table is empty"); const casacore::ScalarColumn polIdColumn( ddTable, casacore::MSDataDescription::columnName( casacore::MSDataDescription::POLARIZATION_ID)); const int polarizationId = polIdColumn(0); for (size_t row = 0; row != ddTable.nrow(); ++row) { if (polIdColumn(row) != polarizationId) throw std::runtime_error( "This measurement set has different polarizations listed in the " "datadescription table. This is non-standard, and AOFlagger cannot " "handle it."); } const casacore::Table polTable = ms.polarization(); const casacore::ArrayColumn corTypeColumn(polTable, "CORR_TYPE"); casacore::Array corType = corTypeColumn(polarizationId); const casacore::Array::iterator iterend(corType.end()); for (casacore::Array::iterator iter = corType.begin(); iter != iterend; ++iter) { const aocommon::PolarizationEnum polarization = aocommon::Polarization::AipsIndexToEnum(*iter); _polarizations.push_back(polarization); } } } /** * Returns an estimate of the size of the data of the MS. * * One thing that might be problematic, is that this function isn't aware of * different spectral windows ("SPW") or different scans. Unlike Dp3, AOFlagger * is used also for telescopes like JVLA that don't necessarily store all times * together. The readers do support it; the unit of a single contiguous stream * of data is called a sequence there. In rare occassions, the timesteps might * not divide the measurement set "linearly" in those cases (e.g. the scan may * use 10 Spws, whereas the second one uses 1, and both might have the same nr * of timesteps -- in which case the second one would take only 1/11th of the * obs). This is a bit more rare though, and it is after all only an estimate, * so maybe we can live with this until we get a bug report :). */ static uint64_t GetMeasurementSetDataSize(casacore::MeasurementSet& ms) { const casacore::MSSpectralWindow spwTable = ms.spectralWindow(); const casacore::ScalarColumn numChanCol( spwTable, casacore::MSSpectralWindow::columnName( casacore::MSSpectralWindowEnums::NUM_CHAN)); const size_t channelCount = numChanCol.get(0); if (channelCount == 0) throw std::runtime_error("No channels in set"); if (ms.nrow() == 0) throw std::runtime_error("Table has no rows (no data)"); typedef float num_t; typedef std::complex complex_t; const casacore::ScalarColumn ant1Column( ms, ms.columnName(casacore::MSMainEnums::ANTENNA1)); const casacore::ScalarColumn ant2Column( ms, ms.columnName(casacore::MSMainEnums::ANTENNA2)); const casacore::ArrayColumn dataColumn( ms, ms.columnName(casacore::MSMainEnums::DATA)); casacore::IPosition dataShape = dataColumn.shape(0); const unsigned polarizationCount = dataShape[0]; return (uint64_t)polarizationCount * (uint64_t)channelCount * (uint64_t)ms.nrow() * (uint64_t)(sizeof(num_t) * 2 + sizeof(bool)); } uint64_t BaselineReader::MeasurementSetDataSize(const string& filename) { casacore::MeasurementSet ms(filename); return GetMeasurementSetDataSize(ms); } static size_t GetNTimeSteps(casacore::MeasurementSet& ms) { size_t result = 0; double time = -1.0; const casacore::ScalarColumn time_column{ ms, ms.columnName(casacore::MSMainEnums::TIME)}; for (size_t i = 0; i != ms.nrow(); ++i) if (const double t = time_column(i); t != time) { ++result; time = t; } return result; } static uint64_t GetMeasurementSetIntervalDataSize(const string& filename, size_t start, size_t end) { assert(start <= end && "Reqested begin and end aren't a valid range"); casacore::MeasurementSet ms(filename); const uint64_t result = GetMeasurementSetDataSize(ms); const size_t time_steps_ms = GetNTimeSteps(ms); const size_t time_steps_requested = end - start; // When more time steps are requested than available use the available number. // Validate time_steps_ms to avoid a division by zero. if (time_steps_requested >= time_steps_ms || time_steps_ms == 0) return result; return result * static_cast(time_steps_requested) / time_steps_ms; } uint64_t BaselineReader::MeasurementSetIntervalDataSize( const string& filename, std::optional start, std::optional end) { if (start) { assert(end && "The engagement status of start and end should be the same."); return GetMeasurementSetIntervalDataSize(filename, *start, *end); } return BaselineReader::MeasurementSetDataSize(filename); } aoflagger-v3.5.1/msio/directbaselinereader.cpp0000664000175000017500000003205214752462134017604 0ustar oleole#include "directbaselinereader.h" #include "msselection.h" #include "../structures/timefrequencydata.h" #include "../util/logger.h" #include "../util/progress/dummyprogresslistener.h" #include "../util/stopwatch.h" #include #include #include #include #include DirectBaselineReader::DirectBaselineReader(const std::string& msFile) : BaselineReader(msFile), _ms(OpenMS()) {} void DirectBaselineReader::initBaselineCache() { // Pass one time through the entire measurement set and store the rownumbers // of the baselines. Logger::Debug << "Determining sequence positions within file for direct " "baseline reader...\n"; std::vector dataIdToSpw; MetaData().GetDataDescToBandVector(dataIdToSpw); casacore::ScalarColumn antenna1Column(_ms, "ANTENNA1"); casacore::ScalarColumn antenna2Column(_ms, "ANTENNA2"); casacore::ScalarColumn dataDescIdColumn(_ms, "DATA_DESC_ID"); DummyProgressListener progress; // TODO MSSelection msSelection(_ms, ObservationTimesPerSequence(), progress); msSelection.Process( [&](size_t rowIndex, size_t sequenceId, size_t /*timeIndexInSequence*/) { int antenna1 = antenna1Column(rowIndex), antenna2 = antenna2Column(rowIndex), dataDescId = dataDescIdColumn(rowIndex); const int spectralWindow = dataIdToSpw[dataDescId]; addRowToBaselineCache(antenna1, antenna2, spectralWindow, sequenceId, rowIndex); }); } void DirectBaselineReader::addRowToBaselineCache(size_t antenna1, size_t antenna2, size_t spectralWindow, size_t sequenceId, size_t row) { BaselineCacheIndex searchItem; searchItem.antenna1 = antenna1; searchItem.antenna2 = antenna2; searchItem.spectralWindow = spectralWindow; searchItem.sequenceId = sequenceId; const std::map::iterator cacheItemIter = _baselineCache.find(searchItem); if (cacheItemIter == _baselineCache.end()) { BaselineCacheValue cacheValue; cacheValue.rows.push_back(row); _baselineCache.insert(std::make_pair(searchItem, cacheValue)); } else { cacheItemIter->second.rows.push_back(row); } } void DirectBaselineReader::addRequestRows( ReadRequest request, size_t requestIndex, std::vector>& rows) { BaselineCacheIndex searchItem; searchItem.antenna1 = request.antenna1; searchItem.antenna2 = request.antenna2; searchItem.spectralWindow = request.spectralWindow; searchItem.sequenceId = request.sequenceId; auto cacheItemIter = _baselineCache.find(searchItem); if (cacheItemIter != _baselineCache.end()) { const std::vector& cacheRows = cacheItemIter->second.rows; for (const size_t j : cacheRows) rows.emplace_back(j, requestIndex); } } void DirectBaselineReader::addRequestRows( FlagWriteRequest request, size_t requestIndex, std::vector>& rows) { BaselineCacheIndex searchItem; searchItem.antenna1 = request.antenna1; searchItem.antenna2 = request.antenna2; searchItem.spectralWindow = request.spectralWindow; searchItem.sequenceId = request.sequenceId; auto cacheItemIter = _baselineCache.find(searchItem); if (cacheItemIter != _baselineCache.end()) { const std::vector& cacheRows = cacheItemIter->second.rows; for (const size_t j : cacheRows) rows.emplace_back(j, requestIndex); } } void DirectBaselineReader::PerformReadRequests(ProgressListener& progress) { progress.OnStartTask("Reading measurement set"); const Stopwatch stopwatch(true); initializeMeta(); if (_baselineCache.empty()) initBaselineCache(); // Each element contains (row number, corresponding request index) std::vector> rows; for (size_t i = 0; i != _readRequests.size(); ++i) addRequestRows(_readRequests[i], i, rows); std::sort(rows.begin(), rows.end()); _results.resize(_readRequests.size()); for (size_t reqIndex = 0; reqIndex != _readRequests.size(); ++reqIndex) { const ReadRequest& request = _readRequests[reqIndex]; Result& result = _results[reqIndex]; size_t startIndex = request.startIndex, endIndex = request.endIndex, band = request.spectralWindow, channelCount = MetaData().FrequencyCount(band); const size_t width = endIndex - startIndex; for (size_t p = 0; p < Polarizations().size(); ++p) { if (ReadData()) { result._realImages.emplace_back( Image2D::CreateZeroImagePtr(width, channelCount)); result._imaginaryImages.emplace_back( Image2D::CreateZeroImagePtr(width, channelCount)); } if (ReadFlags()) { // The flags should be initialized to true, as a baseline might // miss some time scans that other baselines do have, and these // should be flagged. result._flags.emplace_back( Mask2D::CreateSetMaskPtr(width, channelCount)); } } result._uvw.resize(width); } const casacore::ScalarColumn timeColumn(_ms, "TIME"); const casacore::ArrayColumn weightColumn(_ms, "WEIGHT"); const casacore::ArrayColumn uvwColumn(_ms, "UVW"); const casacore::ArrayColumn flagColumn(_ms, "FLAG"); std::unique_ptr> dataColumn; if (ReadData()) dataColumn.reset( new casacore::ArrayColumn(_ms, DataColumnName())); for (size_t i = 0; i != rows.size(); ++i) { progress.OnProgress(i, rows.size()); const std::pair p = rows[i]; const size_t rowIndex = p.first; const size_t requestIndex = p.second; const double time = timeColumn(rowIndex); const ReadRequest& request = _readRequests[requestIndex]; size_t timeIndex = ObservationTimes(request.sequenceId).find(time)->second, startIndex = request.startIndex, endIndex = request.endIndex, band = request.spectralWindow; const bool timeIsSelected = timeIndex >= startIndex && timeIndex < endIndex; if (ReadData() && timeIsSelected) { // if(BaselineReader::DataKind() == WeightData) // readWeights(requestIndex, timeIndex-startIndex, // MetaData().FrequencyCount(band), weightColumn(rowIndex)); else readTimeData(requestIndex, timeIndex - startIndex, MetaData().FrequencyCount(band), (*dataColumn)(rowIndex)); } if (ReadFlags() && timeIsSelected) { readTimeFlags(requestIndex, timeIndex - startIndex, MetaData().FrequencyCount(band), flagColumn(rowIndex)); } if (timeIsSelected) { casacore::Array arr = uvwColumn(rowIndex); casacore::Array::const_contiter i = arr.cbegin(); _results[requestIndex]._uvw[timeIndex - startIndex].u = *i; ++i; _results[requestIndex]._uvw[timeIndex - startIndex].v = *i; ++i; _results[requestIndex]._uvw[timeIndex - startIndex].w = *i; } } _readRequests.clear(); progress.OnFinish(); } std::vector DirectBaselineReader::ReadUVW(unsigned antenna1, unsigned antenna2, unsigned spectralWindow, unsigned sequenceId) { const Stopwatch stopwatch(true); initializeMeta(); if (_baselineCache.empty()) initBaselineCache(); const std::map& observationTimes = ObservationTimes(sequenceId); // Each element contains (row number, corresponding request index) std::vector> rows; ReadRequest request; request.antenna1 = antenna1; request.antenna2 = antenna2; request.spectralWindow = spectralWindow; request.sequenceId = sequenceId; request.startIndex = 0; request.endIndex = observationTimes.size(); addRequestRows(request, 0, rows); std::sort(rows.begin(), rows.end()); const size_t width = observationTimes.size(); const casacore::ScalarColumn timeColumn(_ms, "TIME"); const casacore::ArrayColumn uvwColumn(_ms, "UVW"); std::vector uvws; uvws.resize(width); for (std::vector>::const_iterator i = rows.begin(); i != rows.end(); ++i) { const size_t rowIndex = i->first; const double time = timeColumn(rowIndex); const size_t timeIndex = observationTimes.find(time)->second; casacore::Array arr = uvwColumn(rowIndex); casacore::Array::const_contiter j = arr.cbegin(); UVW& uvw = uvws[timeIndex]; uvw.u = *j; ++j; uvw.v = *j; ++j; uvw.w = *j; } Logger::Debug << "Read of UVW took: " << stopwatch.ToString() << '\n'; return uvws; } void DirectBaselineReader::PerformFlagWriteRequests() { const Stopwatch stopwatch(true); initializeMeta(); if (_baselineCache.empty()) initBaselineCache(); // Each element contains (row number, corresponding request index) std::vector> rows; for (size_t i = 0; i != _writeRequests.size(); ++i) addRequestRows(_writeRequests[i], i, rows); std::sort(rows.begin(), rows.end()); _ms.reopenRW(); const casacore::ScalarColumn timeColumn(_ms, "TIME"); casacore::ArrayColumn flagColumn(_ms, "FLAG"); for (const FlagWriteRequest& request : _writeRequests) { const size_t band = request.spectralWindow; if (MetaData().FrequencyCount(band) != request.flags[0]->Height()) { std::cerr << "The frequency count in the measurement set (" << MetaData().FrequencyCount(band) << ") does not match the image!\n"; } if (request.endIndex - request.startIndex != request.flags[0]->Width()) { std::cerr << "The number of time scans to write in the measurement set (" << (request.endIndex - request.startIndex) << ") does not match the image (" << request.flags[0]->Width() << ") !\n"; } } size_t rowsWritten = 0; for (const std::pair& row : rows) { const size_t rowIndex = row.first; FlagWriteRequest& request = _writeRequests[row.second]; const double time = timeColumn(rowIndex); const size_t timeIndex = ObservationTimes(request.sequenceId).find(time)->second; if (timeIndex >= request.startIndex + request.leftBorder && timeIndex < request.endIndex - request.rightBorder) { casacore::Array flag = flagColumn(rowIndex); casacore::Array::iterator j = flag.begin(); for (size_t f = 0; f < (size_t)MetaData().FrequencyCount(request.spectralWindow); ++f) { for (size_t p = 0; p < Polarizations().size(); ++p) { *j = request.flags[p]->Value(timeIndex - request.startIndex, f); ++j; } } flagColumn.basePut(rowIndex, flag); ++rowsWritten; } } _writeRequests.clear(); Logger::Debug << rowsWritten << "/" << rows.size() << " rows written in " << stopwatch.ToString() << '\n'; } void DirectBaselineReader::readTimeData( size_t requestIndex, size_t xOffset, size_t frequencyCount, const casacore::Array& data) { casacore::Array::const_contiter i = data.cbegin(); const size_t polarizationCount = Polarizations().size(); for (size_t f = 0; f < (size_t)frequencyCount; ++f) { num_t rv, iv; for (size_t p = 0; p < polarizationCount; ++p) { const casacore::Complex& complex = *i; ++i; rv = complex.real(); iv = complex.imag(); _results[requestIndex]._realImages[p]->SetValue(xOffset, f, rv); _results[requestIndex]._imaginaryImages[p]->SetValue(xOffset, f, iv); } } } void DirectBaselineReader::readTimeFlags(size_t requestIndex, size_t xOffset, size_t frequencyCount, const casacore::Array& flag) { const size_t polarizationCount = Polarizations().size(); casacore::Array::const_iterator j = flag.begin(); for (size_t f = 0; f < (size_t)frequencyCount; ++f) { for (size_t p = 0; p < polarizationCount; ++p) { const bool v = *j; ++j; _results[requestIndex]._flags[p]->SetValue(xOffset, f, v); } } } void DirectBaselineReader::readWeights(size_t requestIndex, size_t xOffset, size_t frequencyCount, const casacore::Array& weight) { const size_t polarizationCount = Polarizations().size(); casacore::Array::const_iterator j = weight.begin(); std::vector values(polarizationCount); for (size_t p = 0; p < polarizationCount; ++p) { values[p] = *j; ++j; } for (size_t f = 0; f < (size_t)frequencyCount; ++f) { for (size_t p = 0; p < polarizationCount; ++p) { _results[requestIndex]._realImages[p]->SetValue(xOffset, f, values[p]); _results[requestIndex]._imaginaryImages[p]->SetValue(xOffset, f, 0.0); } } } aoflagger-v3.5.1/msio/pngfile.cpp0000664000175000017500000000415314752462134015071 0ustar oleole#include "pngfile.h" #include PngFile::PngFile(const std::string& filename, size_t width, size_t height) : _filename(filename), _width(width), _height(height), _pixelSize(4) {} PngFile::~PngFile() {} void PngFile::BeginWrite() { _fp = fopen(_filename.c_str(), "wb"); if (!_fp) throw std::runtime_error("Can not open file"); _png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, (png_voidp) nullptr, nullptr, nullptr); if (!_png_ptr) { fclose(_fp); throw std::runtime_error("Can not create png write structure"); } _info_ptr = png_create_info_struct(_png_ptr); if (!_info_ptr) { png_destroy_write_struct(&_png_ptr, (png_infopp) nullptr); fclose(_fp); throw std::runtime_error("Can not write info structure to file"); } if (setjmp(png_jmpbuf(_png_ptr))) { png_destroy_write_struct(&_png_ptr, &_info_ptr); fclose(_fp); throw std::runtime_error( "Unknown error occured during writing of png file"); } png_init_io(_png_ptr, _fp); png_set_IHDR(_png_ptr, _info_ptr, _width, _height, 8, PNG_COLOR_TYPE_RGB_ALPHA, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT); _row_pointers = (png_bytep*)png_malloc(_png_ptr, _height * sizeof(png_bytep)); for (size_t i = 0; i < _height; i++) _row_pointers[i] = (png_bytep)png_malloc(_png_ptr, _width * _pixelSize); } void PngFile::Close() { png_set_rows(_png_ptr, _info_ptr, _row_pointers); png_write_png(_png_ptr, _info_ptr, PNG_TRANSFORM_IDENTITY, nullptr); png_write_end(_png_ptr, _info_ptr); for (unsigned i = 0; i < _height; i++) png_free(_png_ptr, _row_pointers[i]); png_free(_png_ptr, _row_pointers); png_destroy_write_struct(&_png_ptr, &_info_ptr); fclose(_fp); } void PngFile::Clear(int colorR, int colorG, int colorB, int colorA) { for (size_t y = 0; y < _height; y++) { int xa = 0; for (unsigned x = 0; x < _width; x++) { _row_pointers[y][xa++] = colorR; _row_pointers[y][xa++] = colorG; _row_pointers[y][xa++] = colorB; _row_pointers[y][xa++] = colorA; } } } aoflagger-v3.5.1/msio/rspreader.cpp0000664000175000017500000003256114752462134015440 0ustar oleole#include #include #include #include "rspreader.h" #include "../structures/image2d.h" #include "../structures/mask2d.h" #include "../structures/samplerow.h" #include "../util/logger.h" #include "../util/ffttools.h" const unsigned char RSPReader::BitReverseTable256[256] = { #define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64 #define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16) #define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4) R6(0), R6(2), R6(1), R6(3)}; const unsigned long RSPReader::STATION_INTEGRATION_STEPS = 1024; const unsigned int RSPReader::RCPBeamletData::SIZE = 8; const unsigned int RSPReader::RCPApplicationHeader::SIZE = 16; std::pair RSPReader::ReadChannelBeamlet(unsigned long timestepStart, unsigned long timestepEnd, unsigned beamletCount, unsigned beamletIndex) { const unsigned width = timestepEnd - timestepStart; std::pair data = ReadSingleBeamlet(timestepStart * (unsigned long)256, timestepEnd * (unsigned long)256, beamletCount, beamletIndex); const TimeFrequencyData allX = data.first.Make(aocommon::Polarization::XX); const TimeFrequencyData allY = data.first.Make(aocommon::Polarization::YY); const Image2DCPtr xr = allX.GetRealPart(); const Image2DCPtr xi = allX.GetImaginaryPart(); const Image2DCPtr yr = allY.GetRealPart(); const Image2DCPtr yi = allY.GetImaginaryPart(); const Mask2DCPtr mask = data.first.GetSingleMask(); Image2DPtr outXR = Image2D::CreateUnsetImagePtr(width, 256), outXI = Image2D::CreateUnsetImagePtr(width, 256), outYR = Image2D::CreateUnsetImagePtr(width, 256), outYI = Image2D::CreateUnsetImagePtr(width, 256); const Mask2DPtr outMask = Mask2D::CreateUnsetMaskPtr(width, 256); std::vector observationTimes; for (unsigned long timestep = 0; timestep < timestepEnd - timestepStart; ++timestep) { const unsigned long timestepIndex = timestep * 256; SampleRow realX = SampleRow::MakeFromRow(xr.get(), timestepIndex, 256, 0), imaginaryX = SampleRow::MakeFromRow(xi.get(), timestepIndex, 256, 0), realY = SampleRow::MakeFromRow(yr.get(), timestepIndex, 256, 0), imaginaryY = SampleRow::MakeFromRow(yi.get(), timestepIndex, 256, 0); FFTTools::FFT(realX, imaginaryX); FFTTools::FFT(realY, imaginaryY); realX.SetVerticalImageValues(outXR.get(), timestep); imaginaryX.SetVerticalImageValues(outXI.get(), timestep); realY.SetVerticalImageValues(outYR.get(), timestep); imaginaryY.SetVerticalImageValues(outYI.get(), timestep); observationTimes.push_back( data.second->ObservationTimes()[timestepIndex + 256 / 2]); size_t validValues = 0; for (unsigned y = 0; y < 256; ++y) { if (!mask->Value(timestepIndex + y, 0)) ++validValues; } for (unsigned y = 0; y < 256; ++y) { outMask->SetValue(timestep, y, validValues == 0); } } data.first = TimeFrequencyData(aocommon::Polarization::XX, outXR, outXI, aocommon::Polarization::YY, outYR, outYI); data.first.SetGlobalMask(outMask); BandInfo band = data.second->Band(); band.channels.clear(); for (unsigned i = 0; i < 256; ++i) { ChannelInfo channel; channel.frequencyHz = i + 1; channel.frequencyIndex = i; band.channels.push_back(channel); } data.second->SetBand(band); data.second->SetObservationTimes(observationTimes); return data; } std::pair RSPReader::ReadSingleBeamlet(unsigned long timestepStart, unsigned long timestepEnd, unsigned beamletCount, unsigned beamletIndex) { std::pair data = ReadAllBeamlets(timestepStart, timestepEnd, beamletCount); const unsigned width = timestepEnd - timestepStart; const Image2DPtr realX = Image2D::CreateZeroImagePtr(width, 1); const Image2DPtr imaginaryX = Image2D::CreateZeroImagePtr(width, 1); const Image2DPtr realY = Image2D::CreateZeroImagePtr(width, 1); const Image2DPtr imaginaryY = Image2D::CreateZeroImagePtr(width, 1); const Mask2DPtr mask = Mask2D::CreateUnsetMaskPtr(width, 1); const TimeFrequencyData allX = data.first.Make(aocommon::Polarization::XX); const TimeFrequencyData allY = data.first.Make(aocommon::Polarization::YY); const Image2DCPtr xr = allX.GetRealPart(); const Image2DCPtr xi = allX.GetImaginaryPart(); const Image2DCPtr yr = allY.GetRealPart(); const Image2DCPtr yi = allY.GetImaginaryPart(); const Mask2DCPtr maskWithBeamlets = data.first.GetSingleMask(); for (unsigned x = 0; x < width; ++x) { realX->SetValue(x, 0, xr->Value(x, beamletIndex)); imaginaryX->SetValue(x, 0, xi->Value(x, beamletIndex)); realY->SetValue(x, 0, yr->Value(x, beamletIndex)); imaginaryY->SetValue(x, 0, yi->Value(x, beamletIndex)); mask->SetValue(x, 0, maskWithBeamlets->Value(x, beamletIndex)); } data.first = TimeFrequencyData(aocommon::Polarization::XX, realX, imaginaryX, aocommon::Polarization::YY, realY, imaginaryY); data.first.SetGlobalMask(mask); BandInfo band = data.second->Band(); band.channels[0] = data.second->Band().channels[beamletIndex]; band.channels.resize(1); data.second->SetBand(band); return data; } unsigned long RSPReader::TimeStepCount(size_t beamletCount) const { std::ifstream stream(_rawFile.c_str(), std::ios_base::binary | std::ios_base::in); stream.seekg(0, std::ios_base::end); const unsigned long fileSize = stream.tellg(); stream.seekg(0, std::ios_base::beg); RCPApplicationHeader firstHeader; firstHeader.Read(stream); const unsigned long bytesPerFrame = beamletCount * firstHeader.nofBlocks * RCPBeamletData::SIZE + RCPApplicationHeader::SIZE; const unsigned long frames = fileSize / bytesPerFrame; Logger::Debug << "File has " << frames << " number of frames (" << ((double)(frames * firstHeader.nofBlocks * STATION_INTEGRATION_STEPS) / _clockSpeed) << "s of data)\n"; return frames * firstHeader.nofBlocks; } std::pair RSPReader::ReadAllBeamlets(unsigned long timestepStart, unsigned long timestepEnd, unsigned beamletCount) { const unsigned width = timestepEnd - timestepStart; const Image2DPtr realX = Image2D::CreateZeroImagePtr(width, beamletCount); const Image2DPtr imaginaryX = Image2D::CreateZeroImagePtr(width, beamletCount); const Image2DPtr realY = Image2D::CreateZeroImagePtr(width, beamletCount); const Image2DPtr imaginaryY = Image2D::CreateZeroImagePtr(width, beamletCount); const Mask2DPtr mask = Mask2D::CreateSetMaskPtr(width, beamletCount); std::ifstream file(_rawFile.c_str(), std::ios_base::binary | std::ios_base::in); size_t frame = 0; std::set stations; const TimeFrequencyMetaDataPtr metaData = TimeFrequencyMetaDataPtr(new TimeFrequencyMetaData()); BandInfo band; for (size_t i = 0; i < beamletCount; ++i) { ChannelInfo channel; channel.frequencyHz = i + 1; channel.frequencyIndex = i; band.channels.push_back(channel); } metaData->SetBand(band); std::vector observationTimes; // Read a header and determine the reading start position // Because timestepStart might fall within a block, the RCPApplicationHeader firstHeader; firstHeader.Read(file); const unsigned long bytesPerFrame = beamletCount * firstHeader.nofBlocks * RCPBeamletData::SIZE + RCPApplicationHeader::SIZE; const unsigned long startFrame = timestepStart / (unsigned long)firstHeader.nofBlocks; const unsigned long startByte = startFrame * bytesPerFrame; const unsigned long offsetFromStart = timestepStart - (startFrame * firstHeader.nofBlocks); // Logger::Debug << "Seeking to " << startByte << " (timestepStart=" << // timestepStart << ", offsetFromStart=" << offsetFromStart << ", startFrame=" // << startFrame << ",bytesPerFrame=" << bytesPerFrame << ")\n"; file.seekg(startByte, std::ios_base::beg); // Read the frames unsigned long x = 0; while (x < width + offsetFromStart && file.good()) { RCPApplicationHeader header; header.Read(file); if (header.versionId != 2) { std::stringstream s; s << "Corrupted header found in frame " << frame << "!"; throw std::runtime_error(s.str()); } if (stations.count(header.stationId) == 0) { stations.insert(header.stationId); AntennaInfo antenna; std::stringstream s; s << "LOFAR station with index " << header.stationId; antenna.name = s.str(); metaData->SetAntenna1(antenna); metaData->SetAntenna2(antenna); } for (size_t j = 0; j < beamletCount; ++j) { for (size_t i = 0; i < header.nofBlocks; ++i) { RCPBeamletData data; data.Read(file); if (i + x < width + offsetFromStart && i + x >= offsetFromStart) { const unsigned long pos = i + x - offsetFromStart; realX->SetValue(pos, j, data.xr); imaginaryX->SetValue(pos, j, data.xi); realY->SetValue(pos, j, data.yr); imaginaryY->SetValue(pos, j, data.yi); mask->SetValue(pos, j, false); } } } x += header.nofBlocks; ++frame; } // Logger::Debug << "Read " << frame << " frames.\n"; for (unsigned long i = 0; i < width; ++i) { const unsigned long pos = i + timestepStart; const double time = (double)pos * (double)STATION_INTEGRATION_STEPS / (double)_clockSpeed; observationTimes.push_back(time); } metaData->SetObservationTimes(observationTimes); std::pair data; data.first = TimeFrequencyData(aocommon::Polarization::XX, realX, imaginaryX, aocommon::Polarization::YY, realY, imaginaryY); data.first.SetGlobalMask(mask); data.second = metaData; return data; } void RSPReader::ReadForStatistics(unsigned beamletCount) { const long unsigned timesteps = TimeStepCount(beamletCount); const long unsigned stepSize = 1024; std::vector statistics(beamletCount), timeStartStatistics(beamletCount); std::vector statFile(beamletCount); for (unsigned i = 0; i < beamletCount; ++i) { std::ostringstream str; str << "rsp-statistics" << i << ".txt"; statFile[i] = new std::ofstream(str.str().c_str()); } double startTime = -1.0, periodStartTime = -1.0; for (unsigned long timestepIndex = 0; timestepIndex < timesteps; timestepIndex += stepSize) { // Read the data unsigned long end = timestepIndex + stepSize; if (end > timesteps) end = timesteps; const std::pair dataPair = ReadAllBeamlets(timestepIndex, end, beamletCount); const TimeFrequencyData& data = dataPair.first; if (startTime == -1.0) { startTime = dataPair.second->ObservationTimes()[0]; periodStartTime = startTime; } // Count the statistics for (unsigned imageIndex = 0; imageIndex < data.ImageCount(); ++imageIndex) { const Image2DCPtr image = data.GetImage(imageIndex); for (unsigned y = 0; y < image->Height(); ++y) { for (unsigned x = 0; x < image->Width(); ++x) { int value = (int)image->Value(x, y); if (value < 0) { value = -value; ++(statistics[y].bitUseCount[15]); } unsigned highestBit = (value != 0) ? 1 : 0; for (unsigned bit = 0; bit < 15; ++bit) { if ((value & (2 << bit)) != 0) { highestBit = bit + 1; } } for (unsigned bit = 0; bit < highestBit; ++bit) ++(statistics[y].bitUseCount[bit]); ++(statistics[y].totalCount); } } } if ((timestepIndex / stepSize) % 100000 == 0 || timestepIndex + stepSize >= timesteps) { for (unsigned i = 0; i < beamletCount; ++i) { Logger::Info << "Beamlet index " << i << ":\n"; statistics[i].Print(); } } if ((dataPair.second->ObservationTimes()[0] - periodStartTime) > 60.0) { Logger::Debug << "Processed 1 minute of data (" << (dataPair.second->ObservationTimes()[0] - startTime) << "s)\n"; for (unsigned i = 0; i < beamletCount; ++i) { (*statFile[i]) << (periodStartTime - startTime) << '\t' << (statistics[i].totalCount - timeStartStatistics[i].totalCount); statistics[i].totalCount = timeStartStatistics[i].totalCount; for (unsigned bit = 0; bit < 15; ++bit) { (*statFile[i]) << '\t' << (statistics[i].bitUseCount[bit] - timeStartStatistics[i].bitUseCount[bit]); timeStartStatistics[i].bitUseCount[bit] = statistics[i].bitUseCount[bit]; } (*statFile[i]) << '\n'; } periodStartTime = dataPair.second->ObservationTimes()[0]; } } for (unsigned i = 0; i < beamletCount; ++i) { Logger::Info << "Beamlet index " << i << ":\n"; statistics[i].Print(); delete statFile[i]; } } aoflagger-v3.5.1/.gitmodules0000664000175000017500000000015414771504524014145 0ustar oleole[submodule "external/aocommon"] path = external/aocommon url = https://gitlab.com/aroffringa/aocommon.git aoflagger-v3.5.1/Doxyfile.in0000664000175000017500000030374214752462134014113 0ustar oleole# Doxyfile 1.8.7 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. # # All text after a double hash (##) is considered a comment and is placed in # front of the TAG it is preceding. # # All text after a single hash (#) is considered a comment and will be ignored. # The format is: # TAG = value [value, ...] # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # This tag specifies the encoding used for all characters in the config file # that follow. The default is UTF-8 which is also the encoding used for all text # before the first occurrence of this tag. Doxygen uses libiconv (or the iconv # built into libc) for the transcoding. See http://www.gnu.org/software/libiconv # for the list of possible encodings. # The default value is: UTF-8. DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded by # double-quotes, unless you are using Doxywizard) that should identify the # project for which the documentation is generated. This name is used in the # title of most generated pages and in a few other places. # The default value is: My Project. PROJECT_NAME = "AOFlagger" # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version # control system is used. PROJECT_NUMBER = # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a # quick idea about the purpose of the project. Keep the description short. PROJECT_BRIEF = # With the PROJECT_LOGO tag one can specify an logo or icon that is included in # the documentation. The maximum height of the logo should not exceed 55 pixels # and the maximum width should not exceed 200 pixels. Doxygen will copy the logo # to the output directory. PROJECT_LOGO = # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path # into which the generated documentation will be written. If a relative path is # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. OUTPUT_DIRECTORY = doxygen # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and # will distribute the generated files over these directories. Enabling this # option can be useful when feeding doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise causes # performance problems for the file system. # The default value is: NO. CREATE_SUBDIRS = NO # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode # U+3044. # The default value is: NO. ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. # Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, # Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), # Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, # Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), # Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, # Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, # Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, # Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English # If the BRIEF_MEMBER_DESC tag is set to YES doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. # The default value is: YES. BRIEF_MEMBER_DESC = YES # If the REPEAT_BRIEF tag is set to YES doxygen will prepend the brief # description of a member or function before the detailed description # # Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. # The default value is: YES. REPEAT_BRIEF = YES # This tag implements a quasi-intelligent brief description abbreviator that is # used to form the text in various listings. Each string in this list, if found # as the leading text of the brief description, will be stripped from the text # and the result, after processing the whole list, is used as the annotated # text. Otherwise, the brief description is used as-is. If left blank, the # following values are used ($name is automatically replaced with the name of # the entity):The $name class, The $name widget, The $name file, is, provides, # specifies, contains, represents, a, an and the. ABBREVIATE_BRIEF = # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then # doxygen will generate a detailed section even if there is only a brief # description. # The default value is: NO. ALWAYS_DETAILED_SEC = NO # If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. # The default value is: NO. INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES doxygen will prepend the full path # before files name in the file list and in the header files. If set to NO the # shortest path that makes the file name unique will be used # The default value is: YES. FULL_PATH_NAMES = NO # The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. # Stripping is only done if one of the specified strings matches the left-hand # part of the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which doxygen is run is used as the path to # strip. # # Note that you can specify absolute paths here, but also relative paths, which # will be relative from the directory where doxygen is started. # This tag requires that the tag FULL_PATH_NAMES is set to YES. STRIP_FROM_PATH = # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the # path mentioned in the documentation of a class, which tells the reader which # header file to include in order to use a class. If left blank only the name of # the header file containing the class definition is used. Otherwise one should # specify the list of include paths that are normally passed to the compiler # using the -I flag. STRIP_FROM_INC_PATH = # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but # less readable) file names. This can be useful is your file systems doesn't # support long names like on DOS, Mac, or CD-ROM. # The default value is: NO. SHORT_NAMES = NO # If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the # first line (until the first dot) of a Javadoc-style comment as the brief # description. If set to NO, the Javadoc-style will behave just like regular Qt- # style comments (thus requiring an explicit @brief command for a brief # description.) # The default value is: NO. JAVADOC_AUTOBRIEF = NO # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first # line (until the first dot) of a Qt-style comment as the brief description. If # set to NO, the Qt-style will behave just like regular Qt-style comments (thus # requiring an explicit \brief command for a brief description.) # The default value is: NO. QT_AUTOBRIEF = NO # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a # multi-line C++ special comment block (i.e. a block of //! or /// comments) as # a brief description. This used to be the default behavior. The new default is # to treat a multi-line C++ comment block as a detailed description. Set this # tag to YES if you prefer the old behavior instead. # # Note that setting this tag to YES also means that rational rose comments are # not recognized any more. # The default value is: NO. MULTILINE_CPP_IS_BRIEF = NO # If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the # documentation from any documented member that it re-implements. # The default value is: YES. INHERIT_DOCS = YES # If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce a # new page for each member. If set to NO, the documentation of a member will be # part of the file/class/namespace that contains it. # The default value is: NO. SEPARATE_MEMBER_PAGES = NO # The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen # uses this value to replace tabs by spaces in code fragments. # Minimum value: 1, maximum value: 16, default value: 4. TAB_SIZE = 8 # This tag can be used to specify a number of aliases that act as commands in # the documentation. An alias has the form: # name=value # For example adding # "sideeffect=@par Side Effects:\n" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading # "Side Effects:". You can put \n's in the value part of an alias to insert # newlines. ALIASES = # This tag can be used to specify a number of word-keyword mappings (TCL only). # A mapping has the form "name=value". For example adding "class=itcl::class" # will allow you to use the command class in the itcl::class meaning. TCL_SUBST = # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For # instance, some of the names that are used will be different. The list of all # members will be omitted, etc. # The default value is: NO. OPTIMIZE_OUTPUT_FOR_C = NO # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or # Python sources only. Doxygen will then generate output that is more tailored # for that language. For instance, namespaces will be presented as packages, # qualified scopes will look different, etc. # The default value is: NO. OPTIMIZE_OUTPUT_JAVA = NO # Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran # sources. Doxygen will then generate output that is tailored for Fortran. # The default value is: NO. OPTIMIZE_FOR_FORTRAN = NO # Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL # sources. Doxygen will then generate output that is tailored for VHDL. # The default value is: NO. OPTIMIZE_OUTPUT_VHDL = NO # Doxygen selects the parser to use depending on the extension of the files it # parses. With this tag you can assign which parser to use for a given # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, Javascript, # C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: # FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: # Fortran. In the later case the parser tries to guess whether the code is fixed # or free formatted code, this is the default for Fortran type files), VHDL. For # instance to make doxygen treat .inc files as Fortran files (default is PHP), # and .f files as C (default is Fortran), use: inc=Fortran f=C. # # Note For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise # the files are not read by doxygen. EXTENSION_MAPPING = # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments # according to the Markdown format, which allows for more readable # documentation. See http://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by doxygen, so you can # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in # case of backward compatibilities issues. # The default value is: YES. MARKDOWN_SUPPORT = YES # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by by putting a % sign in front of the word # or globally by setting AUTOLINK_SUPPORT to NO. # The default value is: YES. AUTOLINK_SUPPORT = YES # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want # to include (a tag file for) the STL sources as input, then you should set this # tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); # versus func(std::string) {}). This also make the inheritance and collaboration # diagrams that involve STL classes more complete and accurate. # The default value is: NO. BUILTIN_STL_SUPPORT = YES # If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. # The default value is: NO. CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: # http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen # will parse them like normal C++ but will assume all classes use public instead # of private inheritance when no explicit protection keyword is present. # The default value is: NO. SIP_SUPPORT = NO # For Microsoft's IDL there are propget and propput attributes to indicate # getter and setter methods for a property. Setting this option to YES will make # doxygen to replace the get and set methods by a property in the documentation. # This will only work if the methods are indeed getting or setting a simple # type. If this is not the case, or you want to show the methods anyway, you # should set this option to NO. # The default value is: YES. IDL_PROPERTY_SUPPORT = YES # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC # tag is set to YES, then doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. # The default value is: NO. DISTRIBUTE_GROUP_DOC = NO # Set the SUBGROUPING tag to YES to allow class member groups of the same type # (for instance a group of public functions) to be put as a subgroup of that # type (e.g. under the Public Functions section). Set it to NO to prevent # subgrouping. Alternatively, this can be done per class using the # \nosubgrouping command. # The default value is: YES. SUBGROUPING = YES # When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions # are shown inside the group in which they are included (e.g. using \ingroup) # instead of on a separate page (for HTML and Man pages) or section (for LaTeX # and RTF). # # Note that this feature does not work in combination with # SEPARATE_MEMBER_PAGES. # The default value is: NO. INLINE_GROUPED_CLASSES = NO # When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions # with only public data fields or simple typedef fields will be shown inline in # the documentation of the scope in which they are defined (i.e. file, # namespace, or group documentation), provided this scope is documented. If set # to NO, structs, classes, and unions are shown on a separate page (for HTML and # Man pages) or section (for LaTeX and RTF). # The default value is: NO. INLINE_SIMPLE_STRUCTS = NO # When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or # enum is documented as struct, union, or enum with the name of the typedef. So # typedef struct TypeS {} TypeT, will appear in the documentation as a struct # with name TypeT. When disabled the typedef will appear as a member of a file, # namespace, or class. And the struct will be named TypeS. This can typically be # useful for C code in case the coding convention dictates that all compound # types are typedef'ed and only the typedef is referenced, never the tag name. # The default value is: NO. TYPEDEF_HIDES_STRUCT = NO # The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This # cache is used to resolve symbols given their name and scope. Since this can be # an expensive process and often the same symbol appears multiple times in the # code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small # doxygen will become slower. If the cache is too large, memory is wasted. The # cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range # is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 # symbols. At the end of a run doxygen will report the cache usage and suggest # the optimal cache size from a speed point of view. # Minimum value: 0, maximum value: 9, default value: 0. LOOKUP_CACHE_SIZE = 0 #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- # If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in # documentation are documented, even if no documentation was available. Private # class members and static file members will be hidden unless the # EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. # Note: This will also disable the warnings about undocumented members that are # normally produced when WARNINGS is set to YES. # The default value is: NO. EXTRACT_ALL = NO # If the EXTRACT_PRIVATE tag is set to YES all private members of a class will # be included in the documentation. # The default value is: NO. EXTRACT_PRIVATE = NO # If the EXTRACT_PACKAGE tag is set to YES all members with package or internal # scope will be included in the documentation. # The default value is: NO. EXTRACT_PACKAGE = NO # If the EXTRACT_STATIC tag is set to YES all static members of a file will be # included in the documentation. # The default value is: NO. EXTRACT_STATIC = NO # If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined # locally in source files will be included in the documentation. If set to NO # only classes defined in header files are included. Does not have any effect # for Java sources. # The default value is: YES. EXTRACT_LOCAL_CLASSES = YES # This flag is only useful for Objective-C code. When set to YES local methods, # which are defined in the implementation section but not in the interface are # included in the documentation. If set to NO only methods in the interface are # included. # The default value is: NO. EXTRACT_LOCAL_METHODS = NO # If this flag is set to YES, the members of anonymous namespaces will be # extracted and appear in the documentation as a namespace called # 'anonymous_namespace{file}', where file will be replaced with the base name of # the file that contains the anonymous namespace. By default anonymous namespace # are hidden. # The default value is: NO. EXTRACT_ANON_NSPACES = NO # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation # section is generated. This option has no effect if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO these classes will be included in the various overviews. This option has # no effect if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend # (class|struct|union) declarations. If set to NO these declarations will be # included in the documentation. # The default value is: NO. HIDE_FRIEND_COMPOUNDS = NO # If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any # documentation blocks found inside the body of a function. If set to NO these # blocks will be appended to the function's detailed documentation block. # The default value is: NO. HIDE_IN_BODY_DOCS = NO # The INTERNAL_DOCS tag determines if documentation that is typed after a # \internal command is included. If the tag is set to NO then the documentation # will be excluded. Set it to YES to include the internal documentation. # The default value is: NO. INTERNAL_DOCS = NO # If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file # names in lower-case letters. If set to YES upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows # and Mac users are advised to set this option to NO. # The default value is: system dependent. CASE_SENSE_NAMES = YES # If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with # their full class and namespace scopes in the documentation. If set to YES the # scope will be hidden. # The default value is: NO. HIDE_SCOPE_NAMES = NO # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. SHOW_INCLUDE_FILES = YES # If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each # grouped member an include statement to the documentation, telling the reader # which file to include in order to use the member. # The default value is: NO. SHOW_GROUPED_MEMB_INC = NO # If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include # files with double quotes in the documentation rather than with sharp brackets. # The default value is: NO. FORCE_LOCAL_INCLUDES = NO # If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the # documentation for inline members. # The default value is: YES. INLINE_INFO = YES # If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the # (detailed) documentation of file and class members alphabetically by member # name. If set to NO the members will appear in declaration order. # The default value is: YES. SORT_MEMBER_DOCS = YES # If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief # descriptions of file, namespace and class members alphabetically by member # name. If set to NO the members will appear in declaration order. Note that # this will also influence the order of the classes in the class list. # The default value is: NO. SORT_BRIEF_DOCS = YES # If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the # (brief and detailed) documentation of class members so that constructors and # destructors are listed first. If set to NO the constructors will appear in the # respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. # Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief # member documentation. # Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting # detailed member documentation. # The default value is: NO. SORT_MEMBERS_CTORS_1ST = YES # If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy # of group names into alphabetical order. If set to NO the group names will # appear in their defined order. # The default value is: NO. SORT_GROUP_NAMES = NO # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by # fully-qualified names, including namespaces. If set to NO, the class list will # be sorted only by class name, not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the alphabetical # list. # The default value is: NO. SORT_BY_SCOPE_NAME = NO # If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper # type resolution of all parameters of a function it will reject a match between # the prototype and the implementation of a member function even if there is # only one candidate or it is obvious which candidate to choose by doing a # simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still # accept a match between prototype and implementation in such cases. # The default value is: NO. STRICT_PROTO_MATCHING = NO # The GENERATE_TODOLIST tag can be used to enable ( YES) or disable ( NO) the # todo list. This list is created by putting \todo commands in the # documentation. # The default value is: YES. GENERATE_TODOLIST = YES # The GENERATE_TESTLIST tag can be used to enable ( YES) or disable ( NO) the # test list. This list is created by putting \test commands in the # documentation. # The default value is: YES. GENERATE_TESTLIST = YES # The GENERATE_BUGLIST tag can be used to enable ( YES) or disable ( NO) the bug # list. This list is created by putting \bug commands in the documentation. # The default value is: YES. GENERATE_BUGLIST = YES # The GENERATE_DEPRECATEDLIST tag can be used to enable ( YES) or disable ( NO) # the deprecated list. This list is created by putting \deprecated commands in # the documentation. # The default value is: YES. GENERATE_DEPRECATEDLIST= YES # The ENABLED_SECTIONS tag can be used to enable conditional documentation # sections, marked by \if ... \endif and \cond # ... \endcond blocks. ENABLED_SECTIONS = # The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the # initial value of a variable or macro / define can have for it to appear in the # documentation. If the initializer consists of more lines than specified here # it will be hidden. Use a value of 0 to hide initializers completely. The # appearance of the value of individual variables and macros / defines can be # controlled using \showinitializer or \hideinitializer command in the # documentation regardless of this setting. # Minimum value: 0, maximum value: 10000, default value: 30. MAX_INITIALIZER_LINES = 30 # Set the SHOW_USED_FILES tag to NO to disable the list of files generated at # the bottom of the documentation of classes and structs. If set to YES the list # will mention the files that were used to generate the documentation. # The default value is: YES. SHOW_USED_FILES = YES # Set the SHOW_FILES tag to NO to disable the generation of the Files page. This # will remove the Files entry from the Quick Index and from the Folder Tree View # (if specified). # The default value is: YES. SHOW_FILES = YES # Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces # page. This will remove the Namespaces entry from the Quick Index and from the # Folder Tree View (if specified). # The default value is: YES. SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via # popen()) the command command input-file, where command is the value of the # FILE_VERSION_FILTER tag, and input-file is the name of an input file provided # by doxygen. Whatever the program writes to standard output is used as the file # version. For an example see the documentation. FILE_VERSION_FILTER = # The LAYOUT_FILE tag can be used to specify a layout file which will be parsed # by doxygen. The layout file controls the global structure of the generated # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml # will be used as the name of the layout file. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE # tag is left empty. LAYOUT_FILE = # The CITE_BIB_FILES tag can be used to specify one or more bib files containing # the reference definitions. This must be a list of .bib files. The .bib # extension is automatically appended if omitted. This requires the bibtex tool # to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. # For LaTeX the style of the bibliography can be controlled using # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the # search path. Do not use file names with spaces, bibtex cannot handle them. See # also \cite for info how to create references. CITE_BIB_FILES = #--------------------------------------------------------------------------- # Configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated to # standard output by doxygen. If QUIET is set to YES this implies that the # messages are off. # The default value is: NO. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated to standard error ( stderr) by doxygen. If WARNINGS is set to YES # this implies that the warnings are on. # # Tip: Turn warnings on while writing the documentation. # The default value is: YES. WARNINGS = YES # If the WARN_IF_UNDOCUMENTED tag is set to YES, then doxygen will generate # warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag # will automatically be disabled. # The default value is: YES. WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some parameters # in a documented function, or documenting parameters that don't exist or using # markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return # value. If set to NO doxygen will only warn about wrong or incomplete parameter # documentation, but not about the absence of documentation. # The default value is: NO. WARN_NO_PARAMDOC = NO # The WARN_FORMAT tag determines the format of the warning messages that doxygen # can produce. The string should contain the $file, $line, and $text tags, which # will be replaced by the file and line number from which the warning originated # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard # error (stderr). WARN_LOGFILE = #--------------------------------------------------------------------------- # Configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag is used to specify the files and/or directories that contain # documented source files. You may enter file names like myfile.cpp or # directories like /usr/src/myproject. Separate the files or directories with # spaces. # Note: If this tag is empty the current directory is searched. INPUT = @CMAKE_CURRENT_SOURCE_DIR@/interface/aoflagger.h @CMAKE_CURRENT_SOURCE_DIR@/interface/interface.dox # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv # documentation (see: http://www.gnu.org/software/libiconv) for the list of # possible encodings. # The default value is: UTF-8. INPUT_ENCODING = UTF-8 # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. If left blank the # following patterns are tested:*.c, *.cc, *.cxx, *.cpp, *.c++, *.java, *.ii, # *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp, # *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown, # *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf, # *.qsf, *.as and *.js. FILE_PATTERNS = *.h \ *.dox # The RECURSIVE tag can be used to specify whether or not subdirectories should # be searched for input files as well. # The default value is: NO. RECURSIVE = YES # The EXCLUDE tag can be used to specify files and/or directories that should be # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. # # Note that relative paths are relative to the directory from which doxygen is # run. EXCLUDE = # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded # from the input. # The default value is: NO. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude # certain files from those directories. # # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* EXCLUDE_PATTERNS = # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, # AClass::ANamespace, ANamespace::*Test # # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories use the pattern */test/* EXCLUDE_SYMBOLS = # The EXAMPLE_PATH tag can be used to specify one or more files or directories # that contain example code fragments that are included (see the \include # command). EXAMPLE_PATH = # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and # *.h) to filter out the source-files in the directories. If left blank all # files are included. EXAMPLE_PATTERNS = # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude commands # irrespective of the value of the RECURSIVE tag. # The default value is: NO. EXAMPLE_RECURSIVE = NO # The IMAGE_PATH tag can be used to specify one or more files or directories # that contain images that are to be included in the documentation (see the # \image command). IMAGE_PATH = # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command: # # # # where is the value of the INPUT_FILTER tag, and is the # name of an input file. Doxygen will then use the output that the filter # program writes to standard output. If FILTER_PATTERNS is specified, this tag # will be ignored. # # Note that the filter must not add or remove lines; it is applied before the # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern # basis. Doxygen will compare the file name with each pattern and apply the # filter if there is a match. The filters are a list of the form: pattern=filter # (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how # filters are used. If the FILTER_PATTERNS tag is empty or if none of the # patterns match the file name, INPUT_FILTER is applied. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using # INPUT_FILTER ) will also be used to filter the input files that are used for # producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). # The default value is: NO. FILTER_SOURCE_FILES = NO # The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file # pattern. A pattern will override the setting for FILTER_PATTERN (if any) and # it is also possible to disable source filtering for a specific pattern using # *.ext= (so without naming a filter). # This tag requires that the tag FILTER_SOURCE_FILES is set to YES. FILTER_SOURCE_PATTERNS = # If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that # is part of the input, its contents will be placed on the main page # (index.html). This can be useful if you have a project on for instance GitHub # and want to reuse the introduction page also for the doxygen output. USE_MDFILE_AS_MAINPAGE = #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- # If the SOURCE_BROWSER tag is set to YES then a list of source files will be # generated. Documented entities will be cross-referenced with these sources. # # Note: To get rid of all source code in the generated output, make sure that # also VERBATIM_HEADERS is set to NO. # The default value is: NO. SOURCE_BROWSER = NO # Setting the INLINE_SOURCES tag to YES will include the body of functions, # classes and enums directly into the documentation. # The default value is: NO. INLINE_SOURCES = NO # Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any # special comment blocks from generated source code fragments. Normal C, C++ and # Fortran comments will always remain visible. # The default value is: YES. STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES then for each documented # function all documented functions referencing it will be listed. # The default value is: NO. REFERENCED_BY_RELATION = NO # If the REFERENCES_RELATION tag is set to YES then for each documented function # all documented entities called/used by that function will be listed. # The default value is: NO. REFERENCES_RELATION = NO # If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set # to YES, then the hyperlinks from functions in REFERENCES_RELATION and # REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will # link to the documentation. # The default value is: YES. REFERENCES_LINK_SOURCE = YES # If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the # source code will show a tooltip with additional information such as prototype, # brief description and links to the definition and documentation. Since this # will make the HTML file larger and loading of large files a bit slower, you # can opt to disable this feature. # The default value is: YES. # This tag requires that the tag SOURCE_BROWSER is set to YES. SOURCE_TOOLTIPS = YES # If the USE_HTAGS tag is set to YES then the references to source code will # point to the HTML generated by the htags(1) tool instead of doxygen built-in # source browser. The htags tool is part of GNU's global source tagging system # (see http://www.gnu.org/software/global/global.html). You will need version # 4.8.6 or higher. # # To use it do the following: # - Install the latest version of global # - Enable SOURCE_BROWSER and USE_HTAGS in the config file # - Make sure the INPUT points to the root of the source tree # - Run doxygen as normal # # Doxygen will invoke htags (and that will in turn invoke gtags), so these # tools must be available from the command line (i.e. in the search path). # # The result: instead of the source browser generated by doxygen, the links to # source code will now point to the output of htags. # The default value is: NO. # This tag requires that the tag SOURCE_BROWSER is set to YES. USE_HTAGS = NO # If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a # verbatim copy of the header file for each class for which an include is # specified. Set to NO to disable this. # See also: Section \class. # The default value is: YES. VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- # Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- # If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all # compounds will be generated. Enable this if the project contains a lot of # classes, structs, unions or interfaces. # The default value is: YES. ALPHABETICAL_INDEX = YES # The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in # which the alphabetical index list will be split. # Minimum value: 1, maximum value: 20, default value: 5. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. COLS_IN_ALPHA_INDEX = 5 # In case all classes in a project start with a common prefix, all classes will # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag # can be used to specify a prefix (or a list of prefixes) that should be ignored # while generating the index headers. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES doxygen will generate HTML output # The default value is: YES. GENERATE_HTML = YES # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of # it. # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_OUTPUT = html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). # The default value is: .html. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a user-defined HTML header file for # each generated HTML page. If the tag is left blank doxygen will generate a # standard header. # # To get valid HTML the header file that includes any scripts and style sheets # that doxygen needs, which is dependent on the configuration options used (e.g. # the setting GENERATE_TREEVIEW). It is highly recommended to start with a # default header using # doxygen -w html new_header.html new_footer.html new_stylesheet.css # YourConfigFile # and then modify the file new_header.html. See also section "Doxygen usage" # for information on how to generate the default header that doxygen normally # uses. # Note: The header is subject to change so you typically have to regenerate the # default header when upgrading to a newer version of doxygen. For a description # of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_HEADER = # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each # generated HTML page. If the tag is left blank doxygen will generate a standard # footer. See HTML_HEADER for more information on how to generate a default # footer and what special commands can be used inside the footer. See also # section "Doxygen usage" for information on how to generate the default footer # that doxygen normally uses. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FOOTER = # The HTML_STYLESHEET tag can be used to specify a user-defined cascading style # sheet that is used by each HTML page. It can be used to fine-tune the look of # the HTML output. If left blank doxygen will generate a default style sheet. # See also section "Doxygen usage" for information on how to generate the style # sheet that doxygen normally uses. # Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as # it is more robust and this tag (HTML_STYLESHEET) will in the future become # obsolete. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_STYLESHEET = # The HTML_EXTRA_STYLESHEET tag can be used to specify an additional user- # defined cascading style sheet that is included after the standard style sheets # created by doxygen. Using this option one can overrule certain style aspects. # This is preferred over using HTML_STYLESHEET since it does not replace the # standard style sheet and is therefor more robust against future updates. # Doxygen will copy the style sheet file to the output directory. For an example # see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_STYLESHEET = # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or # other source files which should be copied to the HTML output directory. Note # that these files will be copied to the base HTML output directory. Use the # $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these # files. In the HTML_STYLESHEET file, use the file name only. Also note that the # files will be copied as-is; there are no commands or markers available. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the stylesheet and background images according to # this color. Hue is specified as an angle on a colorwheel, see # http://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. # Minimum value: 0, maximum value: 359, default value: 220. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors # in the HTML output. For a value of 0 the output will use grayscales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_SAT = 100 # The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the # luminance component of the colors in the HTML output. Values below 100 # gradually make the output lighter, whereas values above 100 make the output # darker. The value divided by 100 is the actual gamma applied, so 80 represents # a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not # change the gamma. # Minimum value: 40, maximum value: 240, default value: 80. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_GAMMA = 80 # If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML # page will contain the date and time when the page was generated. Setting this # to NO can help when comparing the output of multiple runs. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_TIMESTAMP = YES # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the # page has loaded. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_DYNAMIC_SECTIONS = NO # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand # and collapse entries dynamically later on. Doxygen will expand the tree to # such a level that at most the specified number of entries are visible (unless # a fully collapsed tree already exceeds this amount). So setting the number of # entries 1 will produce a full collapsed tree by default. 0 is a special value # representing an infinite number of entries and will result in a full expanded # tree by default. # Minimum value: 0, maximum value: 9999, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development # environment (see: http://developer.apple.com/tools/xcode/), introduced with # OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a # Makefile in the HTML output directory. Running make will produce the docset in # that directory and running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at # startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html # for more information. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_DOCSET = NO # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider # (such as a company or product suite) can be grouped. # The default value is: Doxygen generated docs. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_FEEDNAME = "Doxygen generated docs" # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_BUNDLE_ID = org.doxygen.Project # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style # string, e.g. com.mycompany.MyDocSet.documentation. # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_PUBLISHER_ID = org.doxygen.Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop # (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on # Windows. # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML # files are now used as the Windows 98 help format, and will replace the old # Windows help format (.hlp) on all Windows platforms in the future. Compressed # HTML files also contain an index, a table of contents, and you can search for # words in the documentation. The HTML workshop also contains a viewer for # compressed HTML files. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_HTMLHELP = NO # The CHM_FILE tag can be used to specify the file name of the resulting .chm # file. You can add a path in front of the file if the result should not be # written to the html output directory. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_FILE = # The HHC_LOCATION tag can be used to specify the location (absolute path # including file name) of the HTML help compiler ( hhc.exe). If non-empty # doxygen will try to run the HTML help compiler on the generated index.hhp. # The file has to be specified with full path. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. HHC_LOCATION = # The GENERATE_CHI flag controls if a separate .chi index file is generated ( # YES) or that it should be included in the master .chm file ( NO). # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. GENERATE_CHI = NO # The CHM_INDEX_ENCODING is used to encode HtmlHelp index ( hhk), content ( hhc) # and project file content. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_INDEX_ENCODING = # The BINARY_TOC flag controls whether a binary table of contents is generated ( # YES) or a normal table of contents ( NO) in the .chm file. Furthermore it # enables the Previous and Next buttons. # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. BINARY_TOC = NO # The TOC_EXPAND flag can be set to YES to add extra items for group members to # the table of contents of the HTML help documentation and to the tree view. # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. TOC_EXPAND = NO # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help # (.qch) of the generated HTML documentation. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_QHP = NO # If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify # the file name of the resulting .qch file. The path specified is relative to # the HTML output folder. # This tag requires that the tag GENERATE_QHP is set to YES. QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace # (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual # Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- # folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom # Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom # Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's filter section matches. Qt Help Project / Filter Attributes (see: # http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_SECT_FILTER_ATTRS = # The QHG_LOCATION tag can be used to specify the location of Qt's # qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the # generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = # If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be # generated, together with the HTML files, they form an Eclipse help plugin. To # install this plugin and make it available under the help contents menu in # Eclipse, the contents of the directory containing the HTML and XML files needs # to be copied into the plugins directory of eclipse. The name of the directory # within the plugins directory should be the same as the ECLIPSE_DOC_ID value. # After copying Eclipse needs to be restarted before the help appears. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_ECLIPSEHELP = NO # A unique identifier for the Eclipse help plugin. When installing the plugin # the directory name containing the HTML and XML files should also have this # name. Each documentation set should have its own identifier. # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. ECLIPSE_DOC_ID = org.doxygen.Project # If you want full control over the layout of the generated HTML pages it might # be necessary to disable the index and replace it with your own. The # DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top # of each HTML page. A value of NO enables the index and the value YES disables # it. Since the tabs in the index contain the same information as the navigation # tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. DISABLE_INDEX = NO # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag # value is set to YES, a side panel will be generated containing a tree-like # index structure (just like the one that is generated for HTML Help). For this # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom stylesheets (see HTML_EXTRA_STYLESHEET) one can # further fine-tune the look of the index. As an example, the default style # sheet generated by doxygen has an example that shows how to put an image at # the root of the tree instead of the PROJECT_NAME. Since the tree basically has # the same information as the tab index, you could consider setting # DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NO # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # # Note that a value of 0 will completely suppress the enum values from appearing # in the overview section. # Minimum value: 0, maximum value: 20, default value: 4. # This tag requires that the tag GENERATE_HTML is set to YES. ENUM_VALUES_PER_LINE = 4 # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used # to set the initial width (in pixels) of the frame in which the tree is shown. # Minimum value: 0, maximum value: 1500, default value: 250. # This tag requires that the tag GENERATE_HTML is set to YES. TREEVIEW_WIDTH = 250 # When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open links to # external symbols imported via tag files in a separate window. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. EXT_LINKS_IN_WINDOW = NO # Use this tag to change the font size of LaTeX formulas included as images in # the HTML documentation. When you change the font size after a successful # doxygen run you need to manually remove any form_*.png images from the HTML # output directory to force them to be regenerated. # Minimum value: 8, maximum value: 50, default value: 10. # This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_FONTSIZE = 10 # Use the FORMULA_TRANPARENT tag to determine whether or not the images # generated for formulas are transparent PNGs. Transparent PNGs are not # supported properly for IE 6.0, but are supported on all modern browsers. # # Note that when changing this option you need to delete any form_*.png files in # the HTML output directory before the changes have effect. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_TRANSPARENT = YES # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see # http://www.mathjax.org) which uses client side Javascript for the rendering # instead of using prerendered bitmaps. Use this if you do not have LaTeX # installed or if you want to formulas look prettier in the HTML output. When # enabled you may also need to install MathJax separately and configure the path # to it using the MATHJAX_RELPATH option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. USE_MATHJAX = NO # When MathJax is enabled you can set the default output format to be used for # the MathJax output. See the MathJax site (see: # http://docs.mathjax.org/en/latest/output.html) for more details. # Possible values are: HTML-CSS (which is slower, but has the best # compatibility), NativeMML (i.e. MathML) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_FORMAT = HTML-CSS # When MathJax is enabled you need to specify the location relative to the HTML # output directory using the MATHJAX_RELPATH option. The destination directory # should contain the MathJax.js script. For instance, if the mathjax directory # is located at the same level as the HTML output directory, then # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of # MathJax from http://www.mathjax.org before deployment. # The default value is: http://cdn.mathjax.org/mathjax/latest. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site # (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_CODEFILE = # When the SEARCHENGINE tag is enabled doxygen will generate a search box for # the HTML output. The underlying search engine uses javascript and DHTML and # should work on any modern browser. Note that when using HTML help # (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) # there is already a search function so this one should typically be disabled. # For large projects the javascript based search engine can be slow, then # enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to # search using the keyboard; to jump to the search box use + S # (what the is depends on the OS and browser, but it is typically # , /