From 948a7e1bdaeb5231be33f0d823cc5bfc35cfcf45 Mon Sep 17 00:00:00 2001 From: Claudio Date: Mon, 8 Jun 2020 12:49:34 +0200 Subject: [PATCH 1/9] first commit: original ontologies_api_ruby_client from V.A. v2.5 --- .gitignore | 5 + Gemfile | 6 + Gemfile.lock | 53 ++++ LICENSE.txt | 26 ++ README.md | 153 ++++++++++ Rakefile | 13 + lib/ontologies_api_client.rb | 16 + lib/ontologies_api_client/analytics.rb | 37 +++ lib/ontologies_api_client/base.rb | 143 +++++++++ lib/ontologies_api_client/collection.rb | 119 ++++++++ lib/ontologies_api_client/config.rb | 81 +++++ lib/ontologies_api_client/http.rb | 285 ++++++++++++++++++ lib/ontologies_api_client/link_explorer.rb | 91 ++++++ .../middleware/faraday-last-updated.rb | 26 ++ .../middleware/faraday-long-requests.rb | 31 ++ .../middleware/faraday-object-cache.rb | 242 +++++++++++++++ .../middleware/faraday-slices.rb | 26 ++ .../middleware/faraday-user-apikey.rb | 26 ++ lib/ontologies_api_client/models/category.rb | 12 + lib/ontologies_api_client/models/class.rb | 78 +++++ lib/ontologies_api_client/models/group.rb | 12 + lib/ontologies_api_client/models/instance.rb | 12 + lib/ontologies_api_client/models/mapping.rb | 36 +++ lib/ontologies_api_client/models/metrics.rb | 13 + lib/ontologies_api_client/models/note.rb | 21 ++ lib/ontologies_api_client/models/ontology.rb | 125 ++++++++ .../models/ontology_submission.rb | 33 ++ lib/ontologies_api_client/models/project.rb | 14 + lib/ontologies_api_client/models/property.rb | 63 ++++ lib/ontologies_api_client/models/reply.rb | 21 ++ lib/ontologies_api_client/models/review.rb | 14 + lib/ontologies_api_client/models/slice.rb | 12 + lib/ontologies_api_client/models/user.rb | 36 +++ lib/ontologies_api_client/read_write.rb | 102 +++++++ .../resource_index/resource_index.rb | 62 ++++ ontologies_api_client.gemspec | 51 ++++ test/benchmark/http.rb | 41 +++ test/console.rb | 6 + test/models/test_collection.rb | 44 +++ test/test_case.rb | 10 + 40 files changed, 2197 insertions(+) create mode 100644 .gitignore create mode 100644 Gemfile create mode 100644 Gemfile.lock create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 Rakefile create mode 100644 lib/ontologies_api_client.rb create mode 100644 lib/ontologies_api_client/analytics.rb create mode 100644 lib/ontologies_api_client/base.rb create mode 100644 lib/ontologies_api_client/collection.rb create mode 100644 lib/ontologies_api_client/config.rb create mode 100644 lib/ontologies_api_client/http.rb create mode 100644 lib/ontologies_api_client/link_explorer.rb create mode 100644 lib/ontologies_api_client/middleware/faraday-last-updated.rb create mode 100644 lib/ontologies_api_client/middleware/faraday-long-requests.rb create mode 100644 lib/ontologies_api_client/middleware/faraday-object-cache.rb create mode 100644 lib/ontologies_api_client/middleware/faraday-slices.rb create mode 100644 lib/ontologies_api_client/middleware/faraday-user-apikey.rb create mode 100644 lib/ontologies_api_client/models/category.rb create mode 100644 lib/ontologies_api_client/models/class.rb create mode 100644 lib/ontologies_api_client/models/group.rb create mode 100644 lib/ontologies_api_client/models/instance.rb create mode 100644 lib/ontologies_api_client/models/mapping.rb create mode 100644 lib/ontologies_api_client/models/metrics.rb create mode 100644 lib/ontologies_api_client/models/note.rb create mode 100644 lib/ontologies_api_client/models/ontology.rb create mode 100644 lib/ontologies_api_client/models/ontology_submission.rb create mode 100644 lib/ontologies_api_client/models/project.rb create mode 100644 lib/ontologies_api_client/models/property.rb create mode 100644 lib/ontologies_api_client/models/reply.rb create mode 100644 lib/ontologies_api_client/models/review.rb create mode 100644 lib/ontologies_api_client/models/slice.rb create mode 100644 lib/ontologies_api_client/models/user.rb create mode 100644 lib/ontologies_api_client/read_write.rb create mode 100644 lib/ontologies_api_client/resource_index/resource_index.rb create mode 100644 ontologies_api_client.gemspec create mode 100644 test/benchmark/http.rb create mode 100644 test/console.rb create mode 100644 test/models/test_collection.rb create mode 100644 test/test_case.rb diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dcf52e4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.idea/ +config/config.rb +*.gem + +.DS_Store diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..b43931f --- /dev/null +++ b/Gemfile @@ -0,0 +1,6 @@ +source 'https://rubygems.org' + +gemspec + +gem 'rake' +gem 'pry' \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..46fa239 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,53 @@ +PATH + remote: . + specs: + ontologies_api_client (0.0.6) + activesupport (~> 4.2.1) + excon + faraday + lz4-ruby + multi_json + oj + spawnling + +GEM + remote: https://rubygems.org/ + specs: + activesupport (4.2.10) + i18n (~> 0.7) + minitest (~> 5.1) + thread_safe (~> 0.3, >= 0.3.4) + tzinfo (~> 1.1) + coderay (1.1.2) + concurrent-ruby (1.0.5) + excon (0.59.0) + faraday (0.13.1) + multipart-post (>= 1.2, < 3) + i18n (0.9.0) + concurrent-ruby (~> 1.0) + lz4-ruby (0.3.3) + method_source (0.9.0) + minitest (5.10.3) + multi_json (1.12.2) + multipart-post (2.0.0) + oj (3.3.9) + pry (0.11.2) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + rake (12.2.1) + spawnling (2.1.6) + thread_safe (0.3.6) + tzinfo (1.2.4) + thread_safe (~> 0.1) + +PLATFORMS + ruby + x86_64-darwin-16 + +DEPENDENCIES + ontologies_api_client! + pry + rake + +BUNDLED WITH + 1.15.4 diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..6ccc739 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,26 @@ +Copyright (c) 2011, The Board of Trustees of Leland Stanford Junior University +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are +permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list + of conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY The Board of Trustees of Leland Stanford Junior University +''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL The Board of Trustees of Leland Stanford Junior University OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those of the +authors and should not be interpreted as representing official policies, either expressed +or implied, of The Board of Trustees of Leland Stanford Junior University. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c5fd2db --- /dev/null +++ b/README.md @@ -0,0 +1,153 @@ +# NCBO Ontologies API Client + +## Install + + gem install ontologies_api_client + +## Configuration + +Configuration is provided by calling the config method + +```ruby +require 'ontologies_api_client' +LinkedData::Client.config do |config| + config.rest_url = "http://stagedata.bioontology.org" + config.apikey = "your_apikey" + config.links_attr = "links" + config.cache = false +end +``` + +## Usage + +Once configured, you can utilize the existing resources that are defined (see lib/ontologies_api_client/models) +to access a resource, its information, and related resources. + +### Retrieval + +There are multiple ways to retrieve individual or groups of resources. + +***Find*** + +To retrieve a single record by id: + +```ruby +Category.find("http://data.bioontology.org/categories/all_organisms") +``` + +***Where*** + +To retrieve all records that match a particular an in-code filter. The code is a block that should return a +boolean that indicates whether or not the item should be included in the results. + +```ruby +categories = Category.where do |ont| + ont.name.include?("health") +end +``` + +***Find By*** + +You can use shortcut methods to find by particular attribute/value pairs +(attributes are named in the method and multiple can be provided by connecting them with 'and'). + +```ruby +categories = Category.find_by_parentCategory("http://data.bioontology.org/categories/anatomy") +``` + +## Create / Update / Delete + +Creates are done via HTTP POST, update via HTTP PATCH, and deletes using HTTP DELETE. + +### Create + +```ruby +ontology_values = { + acronym: "MY_ONT", + name: "My Ontology", + administeredBy: [http://data.bioontology.org/users/my_user] +} +ontology = LinkedData::Client::Models::Ontology.new(values: ontology_values) +response = ontology.save +puts ontology_saved.errors +``` + +### Update + +```ruby +new_values = { + administeredBy: [http://data.bioontology.org/users/my_other_user] +} +ontology = LinkedData::Client::Models::Ontology.find_by_acronym("MY_ONT") +ontology.update_from_params(params[:ontology]) +response = ontology.update +puts response.errors +``` + +### Delete + +```ruby +ontology = LinkedData::Client::Models::Ontology.find_by_acronym("MY_ONT") +response = ontology.delete +``` + +## Hypermedia Navigation + +All resources have a collection of hypermedia links, available by calling the 'links' method. +These links can be navigated by calling the 'explore' method and chaining the link: + +```ruby +ontology = Category.find("http://data.bioontology.org/categories/all_organisms") +classes = ontology.explore.classes +``` + +Links may contain a [URI template](http://tools.ietf.org/html/rfc6570). In this case, the template can be +populated by passing in ordered values for the template tokens: + +```ruby +cls = ontology.explore.single_class("http://my.class.id/class1") +``` + +## Defining Resources + +The client is designed to consume resources from the [NCBO Ontologies API](https://github.com/ncbo/ontologies_api). +Resources are defined in the client using media types that we know about and +providing attribute names that we want to retreive for each media type. + +For example: + +```ruby +class Category < LinkedData::Client::Base + include LinkedData::Client::Collection + @media_type = "http://data.bioontology.org/metadata/Category" +end +``` + +### Collections + +Resources that are available via collections should include the Collection mixin (LinkedData::Client::Collection). +By 'collection', we mean that the all resources are available at a single endpoint. +For example, 'Ontology' is a resource with collections because you can see all ontologgies +at the "/ontologies" URL. + +### Read/Write + +Resources that should have save, update, and delete methods will need to include the ReadWrite mixin (LinkedData::Client::ReadWrite). + +## Questions + +For questions please email [support@bioontology.org](support@bioontology.org.) + +## License + +Copyright (c) 2014, The Board of Trustees of Leland Stanford Junior University All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE BOARD OF TRUSTEES OF LELAND STANFORD JUNIOR UNIVERSITY ''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL The Board of Trustees of Leland Stanford Junior University OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of The Board of Trustees of Leland Stanford Junior University. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..08510a5 --- /dev/null +++ b/Rakefile @@ -0,0 +1,13 @@ +require 'rake/testtask' + +Rake::TestTask.new do |t| + t.libs = [] + t.test_files = FileList['test/**/test*.rb'] +end + +Rake::TestTask.new do |t| + t.libs = [] + t.name = "test:models" + t.test_files = FileList['test/models/test*.rb'] +end + diff --git a/lib/ontologies_api_client.rb b/lib/ontologies_api_client.rb new file mode 100644 index 0000000..8fd6159 --- /dev/null +++ b/lib/ontologies_api_client.rb @@ -0,0 +1,16 @@ +require 'oj' +require 'multi_json' +require 'spawnling' + +require_relative 'ontologies_api_client/config' +require_relative 'ontologies_api_client/http' +require_relative 'ontologies_api_client/link_explorer' +require_relative 'ontologies_api_client/base' +require_relative 'ontologies_api_client/collection' +require_relative 'ontologies_api_client/read_write' +require_relative 'ontologies_api_client/resource_index/resource_index' +require_relative 'ontologies_api_client/analytics' + +# Models +curr_dir = File.expand_path("../ontologies_api_client", __FILE__) +Dir.glob("#{curr_dir}/models/*.rb").each {|f| require_relative f } \ No newline at end of file diff --git a/lib/ontologies_api_client/analytics.rb b/lib/ontologies_api_client/analytics.rb new file mode 100644 index 0000000..cbb85ac --- /dev/null +++ b/lib/ontologies_api_client/analytics.rb @@ -0,0 +1,37 @@ +module LinkedData::Client + class Analytics + HTTP = LinkedData::Client::HTTP + + attr_accessor :onts, :date + + def self.all(params = {}) + get(:analytics) + end + + def self.last_month + data = self.new + data.date = last_month = DateTime.now - 1.month + year_num = last_month.year + month_num = last_month.month + analytics = get(:analytics, {year: year_num, month: month_num}).to_h + analytics.delete(:links) + analytics.delete(:context) + onts = [] + analytics.keys.each do |ont| + views = analytics[ont][:"#{year_num}"][:"#{month_num}"] + onts << {ont: ont, views: views} + end + data.onts = onts + data + end + + private + + def self.get(path, params = {}) + path = path.to_s + path = "/"+path unless path.start_with?("/") + HTTP.get(path, params) + end + + end +end \ No newline at end of file diff --git a/lib/ontologies_api_client/base.rb b/lib/ontologies_api_client/base.rb new file mode 100644 index 0000000..64af912 --- /dev/null +++ b/lib/ontologies_api_client/base.rb @@ -0,0 +1,143 @@ +module LinkedData + module Client + class Base + HTTP = LinkedData::Client::HTTP + attr_writer :instance_values + attr_accessor :context, :links + + class << self + attr_accessor :media_type, :act_as_media_type, :include_attrs, :include_attrs_full, :attrs_always_present + end + + ## + # Passing full: true to explore methods will give you more attributes + def self.attributes(full = false) + if full && @include_attrs_full + @include_attrs_full + else + @include_attrs + end + end + + def self.class_for_type(media_type) + return @media_type_class_map[media_type] if @media_type_class_map + @media_type_class_map = map_classes + return @media_type_class_map[media_type] + end + + def self.map_classes + map = {} + classes = LinkedData::Client::Models.constants.map {|c| LinkedData::Client::Models.const_get(c)} + classes.each do |media_type_cls| + next if map[media_type_cls] || !media_type_cls.respond_to?(:media_type) || !media_type_cls.ancestors.include?(LinkedData::Client::Base) + map[media_type_cls.media_type] = media_type_cls + media_type_cls.act_as_media_type.each {|mt| map[mt] = media_type_cls} if media_type_cls.act_as_media_type + end + return map + end + + def initialize(options = {}) + read_only = options.delete(:read_only) || false + values = options[:values] + if values.is_a?(Hash) && !values.empty? + create_attributes(values.keys) + populate_attributes(values) + end + create_attributes(self.class.attrs_always_present || []) + end + + def id + @id + end + + def type + @type + end + + ## + # Retrieve a set of data using a link provided on an object + # This instantiates an instance of this class and uses + # method missing to determine which link to follow + def explore + LinkedData::Client::LinkExplorer.new(@links, self) + end + + def to_hash + dump = marshal_dump + dump.keys.each do |k| + next unless k.to_s[0].eql?("@") + dump[k[1..-1].to_sym] = dump[k] + dump.delete(k) + end + dump + end + alias :to_param :to_hash + + def to_jsonld + HTTP.get(self.id, {}, {raw: true}) + end + + def marshal_dump + Hash[self.instance_variables.map { |v| [v, self.instance_variable_get("#{v}")] }] + end + + def marshal_load(data) + data ||= {} + create_attributes(data.keys) + populate_attributes(data) + end + + def method_missing(meth, *args, &block) + if meth.to_s[-1].eql?("=") + # This enables OpenStruct-like behavior for setting attributes that aren't defined + attr = meth.to_s.chomp("=").to_sym + create_attributes([attr]) + populate_attributes(attr => args.first) + else + nil + end + end + + def respond_to?(meth, private = false) + true + end + + def [](key) + key = "@#{key}" unless key.to_s.start_with?("@") + instance_variable_get(key) + end + + def []=(key, value) + create_attributes([key]) + populate_attributes({key.to_sym => value}) + end + + private + + def create_attributes(attributes) + attributes.each do |attr| + attr = attr.to_s[1..-1].to_sym if attr.to_s.start_with?("@") + attr_exists = self.public_methods(false).include?(attr) + unless attr_exists + self.class.class_eval do + define_method attr.to_sym do + instance_variable_get("@#{attr}") + end + define_method "#{attr}=" do |val| + instance_variable_set("@#{attr}", val) + end + end + end + end + end + + def populate_attributes(hash) + hash.each do |k,v| + k = "@#{k}" unless k.to_s.start_with?("@") + instance_variable_set(k, v) + end + end + + end + end +end diff --git a/lib/ontologies_api_client/collection.rb b/lib/ontologies_api_client/collection.rb new file mode 100644 index 0000000..92857e4 --- /dev/null +++ b/lib/ontologies_api_client/collection.rb @@ -0,0 +1,119 @@ +require_relative 'config' +require_relative 'http' + +module LinkedData + module Client + module Collection + + def self.included(base) + base.extend(ClassMethods) + end + + module ClassMethods + ## + # Allows for arbitrary find_by methods. For example: + # Ontology.find_by_acronym("BRO") + # Ontology.find_by_group_and_category("UMLS", "Anatomy") + def method_missing(meth, *args, &block) + if meth.to_s =~ /^find_by_(.+)$/ + find_by($1, *args, &block) + else + super + end + end + + ## + # Get all top-level links for the API + def top_level_links + HTTP.get(LinkedData::Client.settings.rest_url) + end + + ## + # Return a link given an object (with links) and a media type + def uri_from_context(object, media_type) + object.links.each do |type, link| + return link if link.media_type && link.media_type.downcase.eql?(media_type.downcase) + end + end + + ## + # Get the first collection of resources for a given type + def entry_point(media_type, params = {}) + params = {include: @include_attrs}.merge(params) + HTTP.get(uri_from_context(top_level_links, media_type), params) + end + + ## + # For a type that is already defined, get the collection path + def collection_path + uri_from_context(top_level_links, @media_type) + end + + ## + # Get all resources from the base collection for a resource + def all(*args) + params = args.shift || {} + entry_point(@media_type, params) + end + + ## + # Get all resources from the base collection for a resource as a hash with resource ids as the keys + def all_to_hash(*args) + all = all(*args) + Hash[all.map {|e| [e.id, e]}] + end + + ## + # Find certain resources from the collection by passing a block that filters results + def where(params = {}, &block) + if block_given? + return all(params).select {|e| block.call(e)} + else + raise ArgumentException("Must provide a block to find items") + end + end + + ## + # Find a resource by id + def find(id, params = {}) + found = where do |obj| + obj.id.eql?(id) + end + found.first + end + + ## + # Get a resource by id (this will retrieve it from the REST service) + def get(id, params = {}) + HTTP.get(id, params) + end + + ## + # Find a resource by a combination of attributes + def find_by(attrs, *args) + attributes = attrs.split("_and_") + values_to_find = args.slice!(0..attributes.length-1) + params = args.shift + unless params.is_a?(Hash) + args.unshift(params) + params = {} + end + where(params) do |obj| + bools = [] + attributes.each_with_index do |attr, index| + if obj.respond_to?(attr) + value = obj.send(attr) + if value.is_a?(Enumerable) + bools << value.include?(values_to_find[index]) + else + bools << (value == values_to_find[index]) + end + end + end + bools.all? + end + end + end + end + end +end diff --git a/lib/ontologies_api_client/config.rb b/lib/ontologies_api_client/config.rb new file mode 100644 index 0000000..59ab403 --- /dev/null +++ b/lib/ontologies_api_client/config.rb @@ -0,0 +1,81 @@ +require 'ostruct' +require 'faraday' +require 'excon' +require 'logger' + +module LinkedData + module Client + extend self + attr_reader :settings + + @settings = OpenStruct.new + @settings_run = false + @settings_run_connection = false + + def config(&block) + return if @settings_run + @settings_run = true + + yield @settings if block_given? + + # Set defaults + @settings.rest_url ||= "http://stagedata.bioontology.org" + @settings.apikey ||= "4ea81d74-8960-4525-810b-fa1baab576ff" + @settings.links_attr ||= "links" + @settings.cache ||= false + @settings.enable_long_request_log ||= false + @settings.purl_prefix ||= "http://purl.example.org" + + # Remove trailing slash + @settings.rest_url = @settings.rest_url.chomp("/") + @settings.purl_prefix = @settings.purl_prefix.chomp("/") + + @settings_run = true + end + + def config_connection(options = {}) + return if @settings_run_connection + store = options[:cache_store] + @settings.conn = Faraday.new(@settings.rest_url) do |faraday| + if @settings.enable_long_request_log + require_relative 'middleware/faraday-long-requests' + faraday.use :long_requests + end + + require_relative 'middleware/faraday-user-apikey' + faraday.use :user_apikey + + require_relative 'middleware/faraday-slices' + faraday.use :ncbo_slices + + require_relative 'middleware/faraday-last-updated' + faraday.use :last_updated + + if @settings.cache + begin + require_relative 'middleware/faraday-object-cache' + faraday.use :object_cache, store: store + puts "=> faraday caching enabled" + puts "=> faraday cache store: #{store.class}" + rescue LoadError + puts "=> WARNING: faraday http cache gem is not available, caching disabled" + end + end + + faraday.request :multipart + faraday.request :url_encoded + faraday.adapter :excon + faraday.headers = { + "Accept" => "application/json", + "Authorization" => "apikey token=#{@settings.apikey}", + "User-Agent" => "NCBO API Ruby Client v0.1.0" + } + end + @settings_run_connection = true + end + + def connection_configured? + @settings_run_connection + end + end +end \ No newline at end of file diff --git a/lib/ontologies_api_client/http.rb b/lib/ontologies_api_client/http.rb new file mode 100644 index 0000000..ef57f9d --- /dev/null +++ b/lib/ontologies_api_client/http.rb @@ -0,0 +1,285 @@ +require 'oj' +require 'multi_json' +require 'digest' +require 'ostruct' + +## +# This monkeypatch makes OpenStruct act like Struct objects +class OpenStruct + alias :each :each_pair + + def member?(key) + @table.key?(key) + end + + def members + @table.keys + end + + def length + @table.keys.length + end + alias :size :length + + def to_a + @table.to_a + end + + def values + @table.values + end + + def values_at(*selectors) + @table.values.values_at(*selectors) + end +end + +module LinkedData + module Client + module HTTP + class Link < String; attr_accessor :media_type; end + + def self.conn + unless LinkedData::Client.connection_configured? + if Kernel.const_defined?("Rails") + rails = Kernel.const_get("Rails") + store = rails.cache if rails.cache + end + LinkedData::Client.config_connection(cache_store: store) + end + LinkedData::Client.settings.conn + end + + def self.get(path, params = {}, options = {}) + headers = options[:headers] || {} + raw = options[:raw] || false # return the unparsed body of the request + params = params.delete_if {|k,v| v == nil || v.to_s.empty?} + params[:ncbo_cache_buster] = Time.now.to_f if raw # raw requests don't get cached to ensure body is available + invalidate_cache = params.delete(:invalidate_cache) || false + + begin + puts "Getting: #{path} with #{params}" if $DEBUG + begin + response = conn.get do |req| + req.url path + req.params = params.dup + req.options[:timeout] = 60 + req.headers.merge(headers) + req.headers[:invalidate_cache] = invalidate_cache + end + rescue Exception => e + params = Faraday::Utils.build_query(params) + path << "?" unless params.empty? || path.include?("?") + raise e, "Problem retrieving:\n#{path}#{params}\n\nError: #{e.message}\n#{e.backtrace.join("\n\t")}" + end + + raise Exception, "Problem retrieving:\n#{path}\n#{response.body}" if response.status >= 500 + + if raw + obj = response.body + elsif response.respond_to?(:parsed_body) && response.parsed_body + obj = response.parsed_body + obj = obj.dup if obj.frozen? + else + obj = recursive_struct(load_json(response.body)) + end + rescue Exception => e + puts "Problem getting #{path}" if $DEBUG + raise e + end + obj + end + + def self.get_batch(paths, params = {}) + responses = [] + if conn.in_parallel? + conn.in_parallel do + paths.each {|p| responses << conn.get(p, params) } + end + else + responses = threaded_request(paths, params) + end + return responses + end + + def self.post(path, obj, options = {}) + file, file_attribute = params_file_handler(obj) + response = conn.post do |req| + req.url path + custom_req(obj, file, file_attribute, req) + end + raise Exception, response.body if response.status >= 500 + if options[:raw] || false # return the unparsed body of the request + return response.body + else + return recursive_struct(load_json(response.body)) + end + end + + def self.put(path, obj) + file, file_attribute = params_file_handler(obj) + response = conn.put do |req| + req.url path + custom_req(obj, file, file_attribute, req) + end + raise Exception, response.body if response.status >= 500 + recursive_struct(load_json(response.body)) + end + + def self.patch(path, obj) + file, file_attribute = params_file_handler(obj) + response = conn.patch do |req| + req.url path + custom_req(obj, file, file_attribute, req) + end + raise Exception, response.body if response.status >= 500 + end + + def self.delete(id) + puts "Deleting #{id}" if $DEBUG + response = conn.delete id + raise Exception, response.body if response.status >= 500 + end + + def self.object_from_json(json) + recursive_struct(load_json(json)) + end + + private + + def self.custom_req(obj, file, file_attribute, req) + req.headers['Content-Type'] = 'application/json' + + if file + # multipart + boundary = "OntologiesAPIMultipartPost" + req.headers['Content-Type'] = "multipart/mixed; boundary=#{boundary}; type=application/json; start=json" + parts = [] + parts << Faraday::Parts::Part.new(boundary, "json\"\r\nContent-Type: \"application/json; charset=UTF-8", MultiJson.dump(obj)) + parts << Faraday::Parts::Part.new(boundary, file_attribute, file) + parts << Faraday::Parts::EpiloguePart.new(boundary) + req.body = Faraday::CompositeReadIO.new(parts) + req.headers["Content-Length"] = req.body.length.to_s + else + # normal + req.body = MultiJson.dump(obj) + end + + true + end + + def self.params_file_handler(params) + return if params.nil? + file, return_attribute = nil, nil + params.dup.each do |attribute, value| + next unless value.is_a?(File) || value.is_a?(Tempfile) || value.is_a?(ActionDispatch::Http::UploadedFile) + filename = value.original_filename + file = Faraday::UploadIO.new(value.path, "text/plain", filename) + return_attribute = attribute + params.delete(attribute) + end + return file, return_attribute + end + + def self.threaded_request(paths, params) + threads = [] + responses = [] + paths.each do |path| + threads << Thread.new do + responses << get(path, params) + end + end + threads.join + responses + end + + def self.recursive_struct(json_obj) + # TODO: Convert dates to date objects + if json_obj.is_a?(Hash) + return if json_obj.empty? + + value_cls = LinkedData::Client::Base.class_for_type(json_obj["@type"]) + links = prep_links(json_obj) # strip links + context = json_obj.delete("@context") # strip context + + # Create a struct with the left-over attributes to store data + attributes = json_obj.keys.map {|k| k.to_sym} + attributes_always_present = value_cls.attrs_always_present || [] rescue [] + attributes = (attributes + attributes_always_present).uniq + + # Add attributes to instance + if value_cls + instance = value_cls.new + attributes.each do |attr| + attr = attr[1..-1] if attr[0].eql?("@") + instance.class.class_eval do + unless method_defined?(attr.to_sym) + define_method attr.to_sym do + instance_variable_get("@#{attr}") + end + end + unless method_defined?("#{attr}=") + define_method "#{attr}=" do |val| + instance_variable_set("@#{attr}", val) + end + end + end + end + + # Create objects for each key/value pair, recursively + json_obj.each do |attr, value| + attr = attr[1..-1] if attr[0].eql?("@") + instance.instance_variable_set("@#{attr}", recursive_struct(value)) + end + else + # Get the struct class + recursive_obj_hash = {links: nil, context: nil} + json_obj.each do |key, value| + recursive_obj_hash[key] = recursive_struct(value) + end + instance = OpenStruct.new(recursive_obj_hash) + end + + # Assign links/context + instance.links = links if links + instance.context = context if context + elsif json_obj.is_a?(Array) + instance = [] + json_obj.each do |value| + instance << recursive_struct(value) + end + else + instance = value_cls ? value_cls.new(values: json_obj) : json_obj + end + instance + end + + def self.prep_links(obj) + links = obj.delete(LinkedData::Client.settings.links_attr) + return if links.nil? + + context = links.delete("@context") + return if context.nil? + links.keys.each do |link_type| + link = Link.new(links[link_type]) + link.media_type = context[link_type] + links[link_type] = link + end + links + end + + def self.load_json(json) + return if json.nil? || json.empty? + begin + MultiJson.load(json) + rescue Exception => e + raise e, "Problem loading json\n#{json}" + end + end + + def self.dump_json(json) + MultiJson.dump(json) + end + end + end +end diff --git a/lib/ontologies_api_client/link_explorer.rb b/lib/ontologies_api_client/link_explorer.rb new file mode 100644 index 0000000..ce77780 --- /dev/null +++ b/lib/ontologies_api_client/link_explorer.rb @@ -0,0 +1,91 @@ +require 'uri' +require_relative 'http' + +module LinkedData + module Client + class LinkExplorer + HTTP = LinkedData::Client::HTTP + + def initialize(links, instance) + @links = links + @instance = instance + end + + def method_missing(meth, *args, &block) + if combined_links.key?(meth.to_s) + explore_link(meth, *args) + elsif meth == :batch + explore_link(args) + else + super + end + end + + def respond_to?(meth, private = false) + if combined_links.key?(meth.to_s) || meth == :batch + return true + else + super + end + end + + def explore_link(*args) + link = combined_links[args.shift.to_s] + params = args.shift + unless params.is_a?(Hash) + args.push(params) + params = {} + end + replacements = args.shift + full_attributes = params.delete(:full) + if link.is_a?(Array) + value_cls = LinkedData::Client::Base.class_for_type(link.first.media_type) + ids = link.map {|l| l.to_s} + value_cls.where {|o| ids.include?(o.id)} + else + url = replace_template_elements(link.to_s, replacements) + value_cls = LinkedData::Client::Base.class_for_type(link.media_type) + params[:include] ||= value_cls.attributes(full_attributes) + HTTP.get(url, params) + end + end + + def combined_links + linkable_attributes.merge(@links) + end + + private + + def replace_template_elements(url, values = []) + return url if values.nil? || values.empty? + values = values.dup + values = [values] unless values.is_a?(Array) + return url.gsub(/(\{.*?\})/) do + URI.escape(values.shift, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]")) + end + end + + def linkable_attributes + linkable = {} + (@instance.context || {}).each do |attr, val| + next unless val.is_a?(Hash) && val["@id"] + links = (@instance.send(attr.to_sym) || []).dup + if links.is_a?(Array) + new_links = [] + links.each do |link| + link = HTTP::Link.new(link) + link.media_type = val["@id"] + new_links << link + end + links = new_links + else + links = HTTP::Link.new(links) + links.media_type = val["@id"] + end + linkable[attr.to_s] = links + end + linkable + end + end + end +end \ No newline at end of file diff --git a/lib/ontologies_api_client/middleware/faraday-last-updated.rb b/lib/ontologies_api_client/middleware/faraday-last-updated.rb new file mode 100644 index 0000000..6c3ee5b --- /dev/null +++ b/lib/ontologies_api_client/middleware/faraday-last-updated.rb @@ -0,0 +1,26 @@ +module Faraday + ## + # This middleware causes Faraday to return + class LastUpdated < Faraday::Middleware + def initialize(app, *arguments) + super(app) + end + + def call(env) + session = Thread.current[:session] + if session && session[:last_updated] + headers = env[:request_headers] + headers["NCBO-Cache"] = session[:last_updated] + env[:request_headers] = headers + end + @app.call(env) + end + + end +end + +if Faraday.respond_to?(:register_middleware) + Faraday.register_middleware last_updated: Faraday::LastUpdated +elsif Faraday::Middleware.respond_to?(:register_middleware) + Faraday::Middleware.register_middleware last_updated: Faraday::LastUpdated +end \ No newline at end of file diff --git a/lib/ontologies_api_client/middleware/faraday-long-requests.rb b/lib/ontologies_api_client/middleware/faraday-long-requests.rb new file mode 100644 index 0000000..5ecf17b --- /dev/null +++ b/lib/ontologies_api_client/middleware/faraday-long-requests.rb @@ -0,0 +1,31 @@ +module Faraday + class LongRequests < Faraday::Middleware + def initialize(app, *arguments) + super(app) + end + + def call(env) + dup.call!(env) + end + + def call!(env) + start = Time.now + data = @app.call(env) + finish = Time.now + if finish - start > 2 + Thread.new do + open("/Users/palexand/tmp/slow_requests.log", "a") do |f| + f.puts "#{finish - start} #{env[:method].to_s.upcase} #{env[:url].to_s}" + end + end + end + data + end + end +end + +if Faraday.respond_to?(:register_middleware) + Faraday.register_middleware long_requests: Faraday::LongRequests +elsif Faraday::Middleware.respond_to?(:register_middleware) + Faraday::Middleware.register_middleware long_requests: Faraday::LongRequests +end \ No newline at end of file diff --git a/lib/ontologies_api_client/middleware/faraday-object-cache.rb b/lib/ontologies_api_client/middleware/faraday-object-cache.rb new file mode 100644 index 0000000..f5e06d3 --- /dev/null +++ b/lib/ontologies_api_client/middleware/faraday-object-cache.rb @@ -0,0 +1,242 @@ +require 'digest/sha1' +require 'active_support/cache' +require 'lz4-ruby' +require_relative '../http' + +module Faraday + class ObjectCacheResponse < Faraday::Response + attr_accessor :parsed_body + end + + ## + # This middleware causes Faraday to return an actual object instead of a response + # This is done so that the object is cached instead of the unparsed json body. + # Otherwise, we have to re-parse the json on every cache hit, which is extrememly costly + # when compared to unmarshaling an object. + class ObjectCache < Faraday::Middleware + def initialize(app, *arguments) + super(app) + + if arguments.last.is_a? Hash + options = arguments.pop + @logger = options.delete(:logger) + else + options = arguments + end + + @store = options[:store] || ActiveSupport::Cache.lookup_store(store, options) + end + + def call(env) + invalidate_cache = env[:request_headers].delete(:invalidate_cache) + + # Add if newer than last modified statement to headers + request_key = cache_key_for(create_request(env)) + last_modified_key = "LM::#{request_key}" + last_retrieved_key = "LR::#{request_key}" + + # If we invalidate the cache, then it forces a clean request + if invalidate_cache + cache_delete(request_key) + cache_delete(last_modified_key) + cache_delete(last_retrieved_key) + env[:request_headers]["Cache-Control"] = "no-cache" + puts "Invalidated key #{request_key}" if enable_debug(request_key) + end + + # If we made the last request within the expiry + if cache_exist?(last_retrieved_key) && cache_exist?(request_key) + puts "Not expired: #{env[:url].to_s}, key #{request_key}" if enable_debug(request_key) + cached_item = cache_read(request_key) + ld_obj = cached_item.is_a?(Hash) && cached_item.key?(:ld_obj) ? cached_item[:ld_obj] : cached_item + env[:status] = 304 + cached_response = ObjectCacheResponse.new(env) + cached_response.parsed_body = ld_obj + return cached_response + end + + last_modified = cache_read(last_modified_key) + headers = env[:request_headers] + puts "last modified: " + last_modified.to_s if last_modified && enable_debug(request_key) + headers['If-Modified-Since'] = last_modified if last_modified + + @app.call(env).on_complete do |response_env| + # Only cache get and head requests + if [:get, :head].include?(response_env[:method]) + puts "Response status for key #{request_key}: " + response_env[:status].to_s if enable_debug(request_key) + last_modified = response_env[:response_headers]["Last-Modified"] + # Generate key using header hash + key = request_key + + # If the last retrieve time is less than expiry + if response_env[:status] == 304 && cache_exist?(key) + stored_obj = cache_read(key) + + # Update if last modified is different + if stored_obj[:last_modified] != last_modified + puts "Updating cache #{response_env[:url].to_s}, key #{request_key}" if enable_debug(request_key) + stored_obj[:last_modified] = last_modified + cache_write(last_modified_key, last_modified) + cache_write(key, stored_obj) + end + + ld_obj = stored_obj[:ld_obj] + else + if response_env[:body].nil? || response_env[:body].empty? + # We got here with an empty body, meaning the object wasn't + # in the cache (weird). So re-do the request. + puts "REDOING REQUEST, NO CACHE ENTRY for #{response_env[:url].to_s}, key #{request_key}" if enable_debug(request_key) + env[:request_headers].delete("If-Modified-Since") + response_env = @app.call(env).env + puts "REDOING REQUEST expiry: #{response_env[:response_headers]["Cache-Control"]}, last_modified: #{last_modified} for key #{request_key}" if enable_debug(request_key) + end + + ld_obj = LinkedData::Client::HTTP.object_from_json(response_env[:body]) + # This stmt was missing in the old code, resulting in repeated calls to REST because object failed to cache + last_modified = response_env[:response_headers]["Last-Modified"] + expiry = response_env[:response_headers]["Cache-Control"].to_s.split("max-age=").last.to_i + puts "Before storing object: expiry: #{expiry}, last_modified: #{last_modified} for key #{request_key}" if enable_debug(request_key) + + if expiry > 0 && last_modified + # This request is cacheable, store it + puts "Storing object: #{response_env[:url].to_s}, key #{request_key}" if enable_debug(request_key) + stored_obj = {last_modified: last_modified, ld_obj: ld_obj} + cache_write(last_modified_key, last_modified) + cache_write(last_retrieved_key, true, expires_in: expiry) + cache_write(key, stored_obj) + end + end + + response = ObjectCacheResponse.new(response_env) + response.parsed_body = ld_obj + return response + end + end + end + + private + + def enable_debug(key) + return true if LinkedData::Client.settings.debug_client && (LinkedData::Client.settings.debug_client_keys.empty? || LinkedData::Client.settings.debug_client_keys.include?(key)) + false + end + + def cache_write(key, obj, *args) + begin + result = @store.write(key, obj, *args) + rescue + puts "Key too large for cache_write: #{key}" if enable_debug(key) + result = nil + end + + if result + return result + else + # This should still get stored in memcache + # keep it in memory, though, because + # marshal/unmarshal is too slow. + # This way memcache will act as a backup + # and you load from there if it isn't + # in memory yet. + @large_object_cache ||= {} + @large_object_cache[key] = obj + cache_write_compressed(key, obj, *args) + return true + end + end + + def cache_read(key) + obj = @store.read(key) + return if obj.nil? + if obj.is_a?(CompressedMemcache) + # Try to get from the large object cache + large_obj = @large_object_cache[key] if @large_object_cache + # Fallback to the memcache version + large_obj ||= cache_read_compressed(key) + obj = large_obj + end + obj.dup + end + + def cache_exist?(key) + @store.exist?(key) + end + + class CompressedMemcache; attr_accessor :key; end + + ## + # Compress cache entry + def cache_write_compressed(key, obj, *args) + compressed = LZ4::compress(Marshal.dump(obj)) + return if compressed.nil? + placeholder = CompressedMemcache.new + placeholder.key = "#{key}::#{(Time.now.to_f * 1000).to_i}::LZ4" + begin + @store.write(key, placeholder) + @store.write(placeholder.key, compressed) + rescue + puts "Key that failed to cache in cache_write_compressed: #{key}" if enable_debug(key) + @store.delete(key) + @store.delete(placeholder.key) + end + end + + ## + # Read compressed cache entry + def cache_read_compressed(key) + obj = @store.read(key) + if obj.is_a?(CompressedMemcache) + begin + uncompressed = LZ4::uncompress(@store.read(obj.key)) + obj = Marshal.load(uncompressed) + rescue StandardError => e + # There is a problem with the stored value, let's remove it so we don't get the error again + @store.delete(key) + @store.delete(obj.key) + raise e + end + end + obj + end + + def cache_delete(key) + @store.delete(key) + end + + # Internal: Generates a String key for a given request object. + # The request object is folded into a sorted Array (since we can't count + # on hashes order on Ruby 1.8), encoded as JSON and digested as a `SHA1` + # string. + # + # Returns the encoded String. + def cache_key_for(request) + array = request.stringify_keys.to_a.sort + Digest::SHA1.hexdigest(Marshal.dump(array)) + end + + # Internal: Creates a new 'Hash' containing the request information. + # + # env - the environment 'Hash' from the Faraday stack. + # + # Returns a 'Hash' containing the ':method', ':url' and 'request_headers' + # entries. + def create_request(env) + request = env.to_hash.slice(:method, :url, :request_headers) + request[:request_headers] = request[:request_headers].dup + request + end + + def clean_request_headers(request) + request[:request_headers].delete("If-Modified-Since") + request[:request_headers].delete("Expect") + request + end + + end +end + +if Faraday.respond_to?(:register_middleware) + Faraday.register_middleware object_cache: Faraday::ObjectCache +elsif Faraday::Middleware.respond_to?(:register_middleware) + Faraday::Middleware.register_middleware object_cache: Faraday::ObjectCache +end diff --git a/lib/ontologies_api_client/middleware/faraday-slices.rb b/lib/ontologies_api_client/middleware/faraday-slices.rb new file mode 100644 index 0000000..201a8fa --- /dev/null +++ b/lib/ontologies_api_client/middleware/faraday-slices.rb @@ -0,0 +1,26 @@ +module Faraday + ## + # This middleware causes Faraday to return + class Slices < Faraday::Middleware + def initialize(app, *arguments) + super(app) + end + + def call(env) + active_slice = Thread.current[:slice] && Thread.current[:slice][:active] + if active_slice + headers = env[:request_headers] + headers["NCBO-Slice"] = Thread.current[:slice][:acronym] + env[:request_headers] = headers + end + @app.call(env) + end + + end +end + +if Faraday.respond_to?(:register_middleware) + Faraday.register_middleware ncbo_slices: Faraday::Slices +elsif Faraday::Middleware.respond_to?(:register_middleware) + Faraday::Middleware.register_middleware ncbo_slices: Faraday::Slices +end \ No newline at end of file diff --git a/lib/ontologies_api_client/middleware/faraday-user-apikey.rb b/lib/ontologies_api_client/middleware/faraday-user-apikey.rb new file mode 100644 index 0000000..08c73cc --- /dev/null +++ b/lib/ontologies_api_client/middleware/faraday-user-apikey.rb @@ -0,0 +1,26 @@ +module Faraday + ## + # This middleware causes Faraday to return + class UserApikey < Faraday::Middleware + def initialize(app, *arguments) + super(app) + end + + def call(env) + user = Thread.current[:session] && Thread.current[:session][:user] ? Thread.current[:session][:user] : nil + if user + headers = env[:request_headers] + headers["Authorization"] = headers["Authorization"] + "&userapikey=" + user.apikey + env[:request_headers] = headers + end + @app.call(env) + end + + end +end + +if Faraday.respond_to?(:register_middleware) + Faraday.register_middleware user_apikey: Faraday::UserApikey +elsif Faraday::Middleware.respond_to?(:register_middleware) + Faraday::Middleware.register_middleware user_apikey: Faraday::UserApikey +end \ No newline at end of file diff --git a/lib/ontologies_api_client/models/category.rb b/lib/ontologies_api_client/models/category.rb new file mode 100644 index 0000000..0f41b59 --- /dev/null +++ b/lib/ontologies_api_client/models/category.rb @@ -0,0 +1,12 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Category < LinkedData::Client::Base + include LinkedData::Client::Collection + @media_type = "http://data.bioontology.org/metadata/Category" + end + end + end +end diff --git a/lib/ontologies_api_client/models/class.rb b/lib/ontologies_api_client/models/class.rb new file mode 100644 index 0000000..e26c9c1 --- /dev/null +++ b/lib/ontologies_api_client/models/class.rb @@ -0,0 +1,78 @@ +require "uri" +require_relative "../base" + +module LinkedData + module Client + module Models + class Class < LinkedData::Client::Base + HTTP = LinkedData::Client::HTTP + @media_type = "http://www.w3.org/2002/07/owl#Class" + @include_attrs = "prefLabel,definition,synonym,obsolete,hasChildren" + @include_attrs_full = "prefLabel,definition,synonym,obsolete,properties,hasChildren,children" + @attrs_always_present = :prefLabel, :definition, :synonym, :obsolete, :properties, :hasChildren, :children + + alias :fullId :id + + # triple store predicate is + def obsolete? + self.obsolete + end + + def prefLabel(options = {}) + if options[:use_html] + if self.obsolete + return "#{@prefLabel}" + else + return "#{@prefLabel}" + end + else + return @prefLabel + end + end + + # TODO: Implement properly + def relation_icon(parent) + return "" if self.explore.ontology.explore.latest_submission.nil? + return "" unless self.explore.ontology.explore.latest_submission.hasOntologyLanguage.eql?("OBO") + non_subclassOf_parent_rel = !self.subClassOf || + (self.subClassOf && (self.subClassOf.include?("http://www.w3.org/2002/07/owl#Thing") || self.subClassOf.include?(parent.id))) + return "" if non_subclassOf_parent_rel + " " + end + + def to_jsonld + HTTP.get(self.links["self"], {}, {raw: true}) + end + + def purl + return "" if self.links.nil? + return self.id if self.id.include?("purl.") + ont = self.explore.ontology + "#{LinkedData::Client.settings.purl_prefix}/#{ont.acronym}?conceptid=#{URI.escape(self.id, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))}" + end + + def ontology + self.explore.ontology + end + + def self.find(id, ontology, params = {}) + ontology = HTTP.get(ontology, params) + ontology.explore.class(URI.escape(id, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))) + end + + def self.search(*args) + query = args.shift + params = args.shift || {} + params[:q] = query + raise ArgumentError, "You must provide a search query: Class.search(query: 'melanoma')" if query.nil? || !query.is_a?(String) + HTTP.post("/search", params) + end + + def expanded? + !self.children.nil? && self.children.length > 0 + end + + end + end + end +end diff --git a/lib/ontologies_api_client/models/group.rb b/lib/ontologies_api_client/models/group.rb new file mode 100644 index 0000000..803b148 --- /dev/null +++ b/lib/ontologies_api_client/models/group.rb @@ -0,0 +1,12 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Group < LinkedData::Client::Base + include LinkedData::Client::Collection + @media_type = "http://data.bioontology.org/metadata/Group" + end + end + end +end diff --git a/lib/ontologies_api_client/models/instance.rb b/lib/ontologies_api_client/models/instance.rb new file mode 100644 index 0000000..cea5edb --- /dev/null +++ b/lib/ontologies_api_client/models/instance.rb @@ -0,0 +1,12 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Instance < LinkedData::Client::Base + include LinkedData::Client::Collection + @media_type = "http://data.bioontology.org/metadata/Instance" + end + end + end +end diff --git a/lib/ontologies_api_client/models/mapping.rb b/lib/ontologies_api_client/models/mapping.rb new file mode 100644 index 0000000..9e8cf1d --- /dev/null +++ b/lib/ontologies_api_client/models/mapping.rb @@ -0,0 +1,36 @@ +require "uri" +require_relative "../base" + +module LinkedData + module Client + module Models + class Mapping < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + + @media_type = "http://data.bioontology.org/metadata/Mapping" + + def self.find(id, params = {}) + HTTP.get(mappings_url_prefix + URI.escape(id, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]")), params) + end + + def delete + HTTP.delete(mappings_url_prefix + URI.escape(self.id, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))) + end + + private + + ## + # This is in a method because the settings are configured after + # the VM initialization, so the rest_url could be null or wrong + def self.mappings_url_prefix + LinkedData::Client.settings.rest_url + "/mappings/" + end + + def mappings_url_prefix + self.class.mappings_url_prefix + end + end + end + end +end diff --git a/lib/ontologies_api_client/models/metrics.rb b/lib/ontologies_api_client/models/metrics.rb new file mode 100644 index 0000000..8c6dec4 --- /dev/null +++ b/lib/ontologies_api_client/models/metrics.rb @@ -0,0 +1,13 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Metrics < LinkedData::Client::Base + include LinkedData::Client::Collection + + @media_type = "http://data.bioontology.org/metadata/Metrics" + end + end + end +end diff --git a/lib/ontologies_api_client/models/note.rb b/lib/ontologies_api_client/models/note.rb new file mode 100644 index 0000000..a24c534 --- /dev/null +++ b/lib/ontologies_api_client/models/note.rb @@ -0,0 +1,21 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Note < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + @media_type = "http://data.bioontology.org/metadata/Note" + + def deletable?(user) + false + end + + def uuid + self.id.split("/").last + end + end + end + end +end diff --git a/lib/ontologies_api_client/models/ontology.rb b/lib/ontologies_api_client/models/ontology.rb new file mode 100644 index 0000000..6149efb --- /dev/null +++ b/lib/ontologies_api_client/models/ontology.rb @@ -0,0 +1,125 @@ +require 'cgi' +require_relative "../base" + +module LinkedData + module Client + module Models + class Ontology < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + + @media_type = "http://data.bioontology.org/metadata/Ontology" + @include_attrs = "all" + + def flat? + self.flat + end + + def private? + viewingRestriction && viewingRestriction.downcase.eql?("private") + end + + def licensed? + viewingRestriction && viewingRestriction.downcase.eql?("licensed") + end + + def viewing_restricted? + private? || licensed? + end + + def view? + viewOf && viewOf.length > 1 + end + + def purl + if self.acronym + "#{LinkedData::Client.settings.purl_prefix}/#{acronym}" + else + "" + end + end + + def access?(user) + return true if !viewing_restricted? + return false if user.nil? + return true if user.admin? + return self.full_acl.any? {|u| u == user.id} + end + + def admin?(user) + return false if user.nil? + return true if user.admin? + return administeredBy.any? {|u| u == user.id} + end + + def invalidate_cache + self.class.all(invalidate_cache: true, include_views: true) + super + end + + # ACL with administrators + def full_acl + ((self.acl || []) + self.administeredBy).uniq + end + + # For use with select lists, always includes the admin by default + def acl_select + select_opts = [] + self.full_acl.each do |userId| + select_opts << [User.get(userId).username, userId] + end + select_opts + end + + ## + # Method to get the property tree for a given ontology + # Gets the properties from the REST API and then returns a tree + def property_tree + properties = Hash[self.explore.properties.map {|p| [p.id, p]}] + properties.keys.each do |key| + prop = properties[key] + prop.parents.each {|par| properties[par].children << prop if properties[par]} + end + roots = properties.values.select {|p| p.parents.empty?} + root = LinkedData::Client::Models::Property.new + root.children = roots + root + end + + ## + # Find a resource by a combination of attributes + # Override to search for views as well by default + # Views get hidden on the REST service unless the `include_views` + # parameter is set to `true` + def self.find_by(attrs, *args) + params = args.shift + if params.is_a?(Hash) + params[:include_views] = params[:include_views] || true + else + # Stick params back and create a new one + args.push({include_views: true}) + end + args.unshift(params) + super(attrs, *args) + end + + ## + # Find a resource by id + # Override to search for views as well by default + # Views get hidden on the REST service unless the `include_views` + # parameter is set to `true` + def find(id, params = {}) + params[:include_views] = params[:include_views] || true + super(id, params) + end + + ## + # Include parameters commonly used with ontologies + def self.include_params + "acronym,administeredBy,group,hasDomain,name,notes,projects,reviews,summaryOnly,viewingRestriction" + end + + end + end + end +end diff --git a/lib/ontologies_api_client/models/ontology_submission.rb b/lib/ontologies_api_client/models/ontology_submission.rb new file mode 100644 index 0000000..c38e58d --- /dev/null +++ b/lib/ontologies_api_client/models/ontology_submission.rb @@ -0,0 +1,33 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class OntologySubmission < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + @media_type = "http://data.bioontology.org/metadata/OntologySubmission" + @include_attrs = "all" + + PRETTY_FORMATS = { + "UMLS" => "RDF/TTL" + } + def pretty_format + PRETTY_FORMATS[self.hasOntologyLanguage] || self.hasOntologyLanguage + end + + ## + # We override the Collection::all call here to make the submissions retrieval more + # efficient. Most of the time we actually only want the most recent submissions. + # Allowing the triplestore to filter these is much faster than doing it in code. + # Passing the `include_status` parameter as `READY` forces this behavior. + def self.all(*args) + params = args.shift || {} + params[:include_status] ||= "READY" + args.unshift(params) + super(*args) + end + end + end + end +end diff --git a/lib/ontologies_api_client/models/project.rb b/lib/ontologies_api_client/models/project.rb new file mode 100644 index 0000000..32cf43e --- /dev/null +++ b/lib/ontologies_api_client/models/project.rb @@ -0,0 +1,14 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Project < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + + @media_type = "http://data.bioontology.org/metadata/Project" + end + end + end +end diff --git a/lib/ontologies_api_client/models/property.rb b/lib/ontologies_api_client/models/property.rb new file mode 100644 index 0000000..459b48d --- /dev/null +++ b/lib/ontologies_api_client/models/property.rb @@ -0,0 +1,63 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Property < LinkedData::Client::Base + @media_type = "http://data.bioontology.org/metadata/Property" + @act_as_media_type = ["http://www.w3.org/2002/07/owl#DatatypeProperty", "http://www.w3.org/2002/07/owl#ObjectProperty", "http://www.w3.org/2002/07/owl#AnnotationProperty"] + @include_attrs = "all" + + attr_accessor :prefLabel, :children + + def self.properties_to_hash(ary) + ary = ary.is_a?(Array) ? ary : [ary] + ary.map {|p| p.to_hash} + end + + def prefLabel(options = {}) + return "" if @label.nil? && @id.nil? + label = @label.first || id_to_label + if options[:use_html] + return "#{label}" + else + return label + end + end + + def id_to_label + @id.split("/").last.split("#").last + end + + def childrenCount + (@children ||= []).length + end + + def children + (@children ||= []) + end + + def expanded? + self.childrenCount > 0 + end + + def to_hash + hash = super + hash[:prefLabel] = self.prefLabel + hash[:childrenCount] = self.childrenCount + if children.length > 0 + hash[:children] = self.children.map {|p| p.to_hash} + end + hash + end + + def ontology + # hack to get ontology, split the submission URI and trust the path hasn't changed + acronym = @submission.split("/")[4] + LinkedData::Client::Models::Ontology.find_by_acronym(acronym).first + end + + end + end + end +end \ No newline at end of file diff --git a/lib/ontologies_api_client/models/reply.rb b/lib/ontologies_api_client/models/reply.rb new file mode 100644 index 0000000..bf35dc7 --- /dev/null +++ b/lib/ontologies_api_client/models/reply.rb @@ -0,0 +1,21 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Reply < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + @media_type = "http://data.bioontology.org/metadata/Reply" + + def deletable?(user) + false + end + + def uuid + self.id.split("/").last + end + end + end + end +end diff --git a/lib/ontologies_api_client/models/review.rb b/lib/ontologies_api_client/models/review.rb new file mode 100644 index 0000000..bd7cae5 --- /dev/null +++ b/lib/ontologies_api_client/models/review.rb @@ -0,0 +1,14 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Review < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + + @media_type = "http://data.bioontology.org/metadata/Review" + end + end + end +end diff --git a/lib/ontologies_api_client/models/slice.rb b/lib/ontologies_api_client/models/slice.rb new file mode 100644 index 0000000..dbfb743 --- /dev/null +++ b/lib/ontologies_api_client/models/slice.rb @@ -0,0 +1,12 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Slice < LinkedData::Client::Base + include LinkedData::Client::Collection + @media_type = "http://data.bioontology.org/metadata/Slice" + end + end + end +end diff --git a/lib/ontologies_api_client/models/user.rb b/lib/ontologies_api_client/models/user.rb new file mode 100644 index 0000000..47c7523 --- /dev/null +++ b/lib/ontologies_api_client/models/user.rb @@ -0,0 +1,36 @@ +require_relative "../base" +require_relative "../http" + +module LinkedData + module Client + module Models + class User < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + + @media_type = "http://data.bioontology.org/metadata/User" + @include_attrs = "all" + + def self.authenticate(user, password) + auth_params = {user: user, password: password, include: "all"} + LinkedData::Client::HTTP.post("#{LinkedData::Client.settings.rest_url}/users/authenticate", auth_params) + end + + def admin? + respond_to?(:role) && role.include?("ADMINISTRATOR") + end + + def invalidate_cache + super + ## IMPORTANT + # We have to invalidate ontologies here because the user could be setting + # custom ontologies. If we don't do this both on the REST and here then + # the UI cache will not update. + Ontology.all(invalidate_cache: true) + Ontology.all(invalidate_cache: true, include: LinkedData::Client::Models::Ontology.include_params) + end + + end + end + end +end diff --git a/lib/ontologies_api_client/read_write.rb b/lib/ontologies_api_client/read_write.rb new file mode 100644 index 0000000..9aaa818 --- /dev/null +++ b/lib/ontologies_api_client/read_write.rb @@ -0,0 +1,102 @@ +require 'active_support/core_ext/hash' + +module LinkedData + module Client + module ReadWrite + HTTP = LinkedData::Client::HTTP + + def save + resp = HTTP.post(self.class.collection_path, self.to_hash) + invalidate_cache() + resp + end + + def update(options = {}) + values = options[:values] || changed_values() + return if values.empty? + resp = HTTP.patch(self.id, values) + invalidate_cache() + resp + end + + def update_from_params(params) + # We want to populate ALL the attributes from the REST + # service so we know that what we're updating is + # actually the full object + all_values = HTTP.get(self.id, include: "all") + all_values.instance_variables.each do |var| + self.send("#{var}=", all_values.instance_variable_get(var)) + end + + # Now we override the retrieved attributes with new ones + params.each do |k,v| + self.send("#{k}=", v) rescue next + end + self + end + + def changed_values + existing = HTTP.get(self.id, include: "all") + changed_attrs = {} + self.instance_variables.each do |var| + var_sym = var[1..-1].to_sym + next if [:id, :type, :links, :context, :created].include?(var_sym) + new_value = self.instance_variable_get(var) + current_value = existing.instance_variable_get(var) + changed_attrs[var_sym] = new_value unless equivalent?(current_value, new_value) + end + changed_attrs + end + + def delete + resp = HTTP.delete(self.id) + invalidate_cache() + resp + end + + private + + def equivalent?(current_value, new_value) + # If we're comparing an existing embedded object + # then use the id for comparison + if current_value.is_a?(LinkedData::Client::Base) + return current_value.id.eql?(new_value) + end + + # Otherwise, do some complex comparing + case new_value + when String + return current_value.to_s.eql?(new_value) + when Array, Hash + new_value = nil if new_value.is_a?(Array) && new_value.empty? && current_value.nil? + if new_value.is_a?(Array) && new_value.first.is_a?(Hash) + clean_current = current_value.map {|e| e.to_h.symbolize_keys.delete_if {|k,v| v.nil?}}.sort_by {|e| e.is_a?(Hash) ? e.values.sort : e} + clean_new = new_value.map {|e| e.symbolize_keys.delete_if {|k,v| v.nil?}}.sort_by {|e| e.is_a?(Hash) ? e.values.sort : e} + return clean_current.eql?(clean_new) rescue clean_current == clean_new + else + return current_value.sort.eql?(new_value.sort) rescue current_value == new_value + end + end + return current_value.eql?(new_value) rescue current_value == new_value + end + + def invalidate_cache + self.class.all(invalidate_cache: true) + HTTP.get(self.id, invalidate_cache: true) if self.id + session = Thread.current[:session] + session[:last_updated] = Time.now.to_f if session + refresh_cache + end + + def refresh_cache + Spawnling.new do + LinkedData::Client::Models::Ontology.all + LinkedData::Client::Models::OntologySubmission.all + LinkedData::Client::Models::User.all + exit + end + end + + end + end +end diff --git a/lib/ontologies_api_client/resource_index/resource_index.rb b/lib/ontologies_api_client/resource_index/resource_index.rb new file mode 100644 index 0000000..f5b5259 --- /dev/null +++ b/lib/ontologies_api_client/resource_index/resource_index.rb @@ -0,0 +1,62 @@ +module LinkedData::Client + class ResourceIndex + HTTP = LinkedData::Client::HTTP + + def self.resources + get(:resources) + end + + def self.counts(classes, params = {}) + classes = classes.is_a?(Array) ? classes : [classes] + params.merge!(class_params_hash(classes)) + get(:counts, params) + end + + def self.annotation_counts() + get("annotation_counts") + end + + def self.documents(resource_id, classes, params = {}) + classes = classes.is_a?(Array) ? classes : [classes] + params.merge!(class_params_hash(classes)) + get("resources/#{resource_id}/search", params) + end + + def self.documents_all_resources(classes, params = {}) + classes = classes.is_a?(Array) ? classes : [classes] + params.merge!(class_params_hash(classes)) + get("resources/search", params) + end + + def self.class_params_hash(classes, escape = false) + ont_classes = {} + classes.each do |c| + ont_id = c.explore.ontology.id + ont_id = escape ? CGI.escape(ont_id) : ont_id + cls_id = escape ? CGI.escape(c.id) : c.id + ont_classes["classes[#{ont_id}]"] ||= [] + ont_classes["classes[#{ont_id}]"] << cls_id + end + ont_classes.keys.each {|ont_id| ont_classes[ont_id] = ont_classes[ont_id].join(",")} + ont_classes + end + + def self.class_params(classes) + ont_classes = class_params_hash(classes, true) + ont_classes.keys.map {|ont_id| "#{ont_id}=#{ont_classes[ont_id]}"}.join("&") + end + + private + + def self.get(path, params = {}) + path = path.to_s + path = "/"+path unless path.start_with?("/") + HTTP.get(base_path + path, params) + end + + def self.base_path + "/resource_index" + end + + end +end \ No newline at end of file diff --git a/ontologies_api_client.gemspec b/ontologies_api_client.gemspec new file mode 100644 index 0000000..d4aee11 --- /dev/null +++ b/ontologies_api_client.gemspec @@ -0,0 +1,51 @@ +# -*- encoding: utf-8 -*- +# stub: ontologies_api_client 0.0.6 ruby lib + +Gem::Specification.new do |s| + s.name = "ontologies_api_client".freeze + s.version = "0.0.6" + + s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version= + s.require_paths = ["lib".freeze] + s.authors = ["Paul R Alexander".freeze] + s.date = "2018-02-21" + s.description = "Models and serializers for ontologies and related artifacts backed by 4store".freeze + s.email = ["palexander@stanford.edu".freeze] + s.files = [".gitignore".freeze, "Gemfile".freeze, "Gemfile.lock".freeze, "LICENSE.txt".freeze, "README.md".freeze, "Rakefile".freeze, "lib/ontologies_api_client.rb".freeze, "lib/ontologies_api_client/analytics.rb".freeze, "lib/ontologies_api_client/base.rb".freeze, "lib/ontologies_api_client/collection.rb".freeze, "lib/ontologies_api_client/config.rb".freeze, "lib/ontologies_api_client/http.rb".freeze, "lib/ontologies_api_client/link_explorer.rb".freeze, "lib/ontologies_api_client/middleware/faraday-last-updated.rb".freeze, "lib/ontologies_api_client/middleware/faraday-long-requests.rb".freeze, "lib/ontologies_api_client/middleware/faraday-object-cache.rb".freeze, "lib/ontologies_api_client/middleware/faraday-slices.rb".freeze, "lib/ontologies_api_client/middleware/faraday-user-apikey.rb".freeze, "lib/ontologies_api_client/models/category.rb".freeze, "lib/ontologies_api_client/models/class.rb".freeze, "lib/ontologies_api_client/models/group.rb".freeze, "lib/ontologies_api_client/models/instance.rb".freeze, "lib/ontologies_api_client/models/mapping.rb".freeze, "lib/ontologies_api_client/models/metrics.rb".freeze, "lib/ontologies_api_client/models/note.rb".freeze, "lib/ontologies_api_client/models/ontology.rb".freeze, "lib/ontologies_api_client/models/ontology_submission.rb".freeze, "lib/ontologies_api_client/models/project.rb".freeze, "lib/ontologies_api_client/models/property.rb".freeze, "lib/ontologies_api_client/models/reply.rb".freeze, "lib/ontologies_api_client/models/review.rb".freeze, "lib/ontologies_api_client/models/slice.rb".freeze, "lib/ontologies_api_client/models/user.rb".freeze, "lib/ontologies_api_client/read_write.rb".freeze, "lib/ontologies_api_client/resource_index/resource_index.rb".freeze, "ontologies_api_client.gemspec".freeze, "test/benchmark/http.rb".freeze, "test/console.rb".freeze, "test/models/test_collection.rb".freeze, "test/test_case.rb".freeze] + s.homepage = "https://github.com/ncbo/ontologies_api_ruby_client".freeze + s.rubygems_version = "2.5.2.2".freeze + s.summary = "This library can be used for interacting with a 4store instance that stores NCBO-based ontology information. Models in the library are based on Goo. Serializers support RDF serialization as Rack Middleware and automatic generation of hypermedia links.".freeze + s.test_files = ["test/benchmark/http.rb".freeze, "test/console.rb".freeze, "test/models/test_collection.rb".freeze, "test/test_case.rb".freeze] + + s.installed_by_version = "2.5.2.2" if s.respond_to? :installed_by_version + + if s.respond_to? :specification_version then + s.specification_version = 4 + + if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then + s.add_runtime_dependency(%q.freeze, [">= 0"]) + s.add_runtime_dependency(%q.freeze, [">= 0"]) + s.add_runtime_dependency(%q.freeze, [">= 0"]) + s.add_runtime_dependency(%q.freeze, [">= 0"]) + s.add_runtime_dependency(%q.freeze, [">= 0"]) + s.add_runtime_dependency(%q.freeze, [">= 0"]) + s.add_runtime_dependency(%q.freeze, ["~> 4.2.1"]) + else + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, ["~> 4.2.1"]) + end + else + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, [">= 0"]) + s.add_dependency(%q.freeze, ["~> 4.2.1"]) + end +end diff --git a/test/benchmark/http.rb b/test/benchmark/http.rb new file mode 100644 index 0000000..56f5a9b --- /dev/null +++ b/test/benchmark/http.rb @@ -0,0 +1,41 @@ +require 'benchmark' + +require_relative '../../lib/ontologies_api_client' +LinkedData::Client.config + +module LinkedData + module Client + class Benchmark + + def test_all_ontologies + 10.times {LinkedData::Client::Models::Ontology.all} + end + + def test_explore_ontologies + onts = LinkedData::Client::Models::Ontology.all + onts.each do |ont| + ont.explore.projects + ont.explore.groups + ont.explore.categories + ont.explore.reviews + end + end + + def test_batch_explore_ontologies + # onts = LinkedData::Client::Models::Ontology.all + # onts.each do |ont| + # projects, groups, categories = LinkedData::Client::HTTP.get_batch([ont.links["projects"], ont.links["groups"], ont.links["categories"]]) + # end + end + + end + end +end + +benchmark = LinkedData::Client::Benchmark.new +benchmark.public_methods(false).each do |method| + time = ::Benchmark.realtime do + benchmark.send(method) + end + puts "#{method}: #{time*1000}ms" +end \ No newline at end of file diff --git a/test/console.rb b/test/console.rb new file mode 100644 index 0000000..7ed9271 --- /dev/null +++ b/test/console.rb @@ -0,0 +1,6 @@ +require 'pry' +require_relative '../lib/ontologies_api_client' +require_relative '../config/config' + +Pry.start binding, :quiet => true +exit \ No newline at end of file diff --git a/test/models/test_collection.rb b/test/models/test_collection.rb new file mode 100644 index 0000000..31b533b --- /dev/null +++ b/test/models/test_collection.rb @@ -0,0 +1,44 @@ +require_relative '../test_case' + +class TestOntology < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + + @media_type = "http://data.bioontology.org/metadata/Ontology" + @include_attrs = "all" +end + +class CollectionTest < LinkedData::Client::TestCase + def test_all + onts = LinkedData::Client::Models::Ontology.all + assert onts.length > 350 + end + + def test_class_for_type + media_type = "http://data.bioontology.org/metadata/Category" + type_cls = LinkedData::Client::Base.class_for_type(media_type) + assert type_cls == LinkedData::Client::Models::Category + end + + def test_find_by + bro = TestOntology.find_by_acronym("BRO") + assert bro.length >= 1 + assert bro.any? {|o| o.acronym.eql?("BRO")} + + onts = TestOntology.find_by_hasDomain_and_doNotUpdate("http://data.bioontology.org/categories/health", true) + assert onts.length >= 1 + + onts = TestOntology.find_by_hasDomain_and_hasDomain("http://data.bioontology.org/categories/phenotype", "http://data.bioontology.org/categories/human") + assert onts.length >= 1 + end + + def test_where + onts = TestOntology.where {|o| o.name.downcase.start_with?("c")} + assert onts.length >= 1 + end + + def test_find + ont = TestOntology.find("http://data.bioontology.org/ontologies/CAMRQ") + assert !ont.nil? + end +end diff --git a/test/test_case.rb b/test/test_case.rb new file mode 100644 index 0000000..d1989b3 --- /dev/null +++ b/test/test_case.rb @@ -0,0 +1,10 @@ +require 'test/unit' +require_relative '../lib/ontologies_api_client' +require_relative '../config/config.rb' + +module LinkedData + module Client + class TestCase < Test::Unit::TestCase + end + end +end \ No newline at end of file From 5ff1fbac1a62bb745e134d54ccd2e3bd34de213f Mon Sep 17 00:00:00 2001 From: Claudio Date: Mon, 8 Jun 2020 12:59:44 +0200 Subject: [PATCH 2/9] added datacite metadata + doi creation --- lib/ontologies_api_client.rb | 1 + lib/ontologies_api_client/base.rb | 19 ++- lib/ontologies_api_client/collection.rb | 1 + lib/ontologies_api_client/http.rb | 34 +++-- lib/ontologies_api_client/models/creator.rb | 12 ++ .../models/creator_identifier.rb | 12 ++ .../models/identifier_request.rb | 13 ++ lib/ontologies_api_client/read_write.rb | 143 +++++++++++++----- lib/ontologies_api_client/utils.rb | 101 +++++++++++++ 9 files changed, 285 insertions(+), 51 deletions(-) create mode 100644 lib/ontologies_api_client/models/creator.rb create mode 100644 lib/ontologies_api_client/models/creator_identifier.rb create mode 100644 lib/ontologies_api_client/models/identifier_request.rb create mode 100644 lib/ontologies_api_client/utils.rb diff --git a/lib/ontologies_api_client.rb b/lib/ontologies_api_client.rb index 8fd6159..02de678 100644 --- a/lib/ontologies_api_client.rb +++ b/lib/ontologies_api_client.rb @@ -2,6 +2,7 @@ require 'multi_json' require 'spawnling' +require_relative 'ontologies_api_client/utils' require_relative 'ontologies_api_client/config' require_relative 'ontologies_api_client/http' require_relative 'ontologies_api_client/link_explorer' diff --git a/lib/ontologies_api_client/base.rb b/lib/ontologies_api_client/base.rb index 64af912..9cf4560 100644 --- a/lib/ontologies_api_client/base.rb +++ b/lib/ontologies_api_client/base.rb @@ -28,6 +28,7 @@ def self.class_for_type(media_type) def self.map_classes map = {} classes = LinkedData::Client::Models.constants.map {|c| LinkedData::Client::Models.const_get(c)} + #LOGGER.debug(" ONTOLOGIES_API_RUBY_CLIENT - LinkedData::Client::Base -> map_classes: \n\nclasses=#{classes.inspect}") classes.each do |media_type_cls| next if map[media_type_cls] || !media_type_cls.respond_to?(:media_type) || !media_type_cls.ancestors.include?(LinkedData::Client::Base) map[media_type_cls.media_type] = media_type_cls @@ -37,13 +38,19 @@ def self.map_classes end def initialize(options = {}) - read_only = options.delete(:read_only) || false - values = options[:values] - if values.is_a?(Hash) && !values.empty? - create_attributes(values.keys) - populate_attributes(values) + #LOGGER.debug("\n\n ============ RUBY API - base.rb - initialize(options) -> options = #{options}") + begin + read_only = options.delete(:read_only) || false + values = options[:values] + if values.is_a?(Hash) && !values.empty? + create_attributes(values.keys) + populate_attributes(values) + end + create_attributes(self.class.attrs_always_present || []) + rescue => e + LOGGER.debug("\n\n ECCEZIONE! RUBY API - base.rb - initialize(options): #{e.message}\n#{e.backtrace.join("\n")}") + raise e end - create_attributes(self.class.attrs_always_present || []) end def id diff --git a/lib/ontologies_api_client/collection.rb b/lib/ontologies_api_client/collection.rb index 92857e4..1345aaa 100644 --- a/lib/ontologies_api_client/collection.rb +++ b/lib/ontologies_api_client/collection.rb @@ -91,6 +91,7 @@ def get(id, params = {}) ## # Find a resource by a combination of attributes def find_by(attrs, *args) + # LOGGER.debug("\n\n ============ RUBY API - collection.rb - find_by") attributes = attrs.split("_and_") values_to_find = args.slice!(0..attributes.length-1) params = args.shift diff --git a/lib/ontologies_api_client/http.rb b/lib/ontologies_api_client/http.rb index ef57f9d..f182aa1 100644 --- a/lib/ontologies_api_client/http.rb +++ b/lib/ontologies_api_client/http.rb @@ -59,6 +59,8 @@ def self.get(path, params = {}, options = {}) begin puts "Getting: #{path} with #{params}" if $DEBUG + + # LOGGER.debug("\n\n RUBY API - LinkedData::Client::HTTP ->Get: #{path} with:\n > params:#{params.inspect}\n > options:#{options.inspect} ") begin response = conn.get do |req| req.url path @@ -103,17 +105,24 @@ def self.get_batch(paths, params = {}) end def self.post(path, obj, options = {}) - file, file_attribute = params_file_handler(obj) - response = conn.post do |req| - req.url path - custom_req(obj, file, file_attribute, req) - end - raise Exception, response.body if response.status >= 500 - if options[:raw] || false # return the unparsed body of the request - return response.body - else - return recursive_struct(load_json(response.body)) + begin + # LOGGER.debug("\n\n RUBY API - LinkedData::Client::HTTP ->post: with:\n > path:#{path} \n > obj:#{obj.inspect}\n > options:#{options.inspect} ") + file, file_attribute = params_file_handler(obj) + response = conn.post do |req| + req.url path + custom_req(obj, file, file_attribute, req) + end + raise Exception, response.body if response.status >= 500 + if options[:raw] || false # return the unparsed body of the request + return response.body + else + return recursive_struct(load_json(response.body)) + end + rescue => e + LOGGER.debug "\n\n\nRUBY API - LinkedData::Client::HTTP ->post: - ECCEZIONE: #{e.message}\n#{e.backtrace.join("\n")}" + raise e end + end def self.put(path, obj) @@ -194,6 +203,7 @@ def self.threaded_request(paths, params) end def self.recursive_struct(json_obj) + # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::HTTP::Links -> recursive_struct:\n\n json_obj=#{json_obj.inspect}") # TODO: Convert dates to date objects if json_obj.is_a?(Hash) return if json_obj.empty? @@ -206,7 +216,8 @@ def self.recursive_struct(json_obj) attributes = json_obj.keys.map {|k| k.to_sym} attributes_always_present = value_cls.attrs_always_present || [] rescue [] attributes = (attributes + attributes_always_present).uniq - + + #LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::HTTP::Links -> recursive_struct:\n\n value_cls=#{value_cls.inspect}") # Add attributes to instance if value_cls instance = value_cls.new @@ -237,6 +248,7 @@ def self.recursive_struct(json_obj) json_obj.each do |key, value| recursive_obj_hash[key] = recursive_struct(value) end + #LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::HTTP::Links -> recursive_struct:\n\n new OpenStruct for recursive_obj_hash=#{recursive_obj_hash.inspect}") instance = OpenStruct.new(recursive_obj_hash) end diff --git a/lib/ontologies_api_client/models/creator.rb b/lib/ontologies_api_client/models/creator.rb new file mode 100644 index 0000000..4a9569a --- /dev/null +++ b/lib/ontologies_api_client/models/creator.rb @@ -0,0 +1,12 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Creator < LinkedData::Client::Base + include LinkedData::Client::ReadWrite + @media_type = "http://data.ecoportal.org/metadata/Creator" + end + end + end +end diff --git a/lib/ontologies_api_client/models/creator_identifier.rb b/lib/ontologies_api_client/models/creator_identifier.rb new file mode 100644 index 0000000..c7da7c8 --- /dev/null +++ b/lib/ontologies_api_client/models/creator_identifier.rb @@ -0,0 +1,12 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class CreatorIdentifier < LinkedData::Client::Base + include LinkedData::Client::ReadWrite + @media_type = "http://data.ecoportal.org/metadata/CreatorIdentifier" + end + end + end +end diff --git a/lib/ontologies_api_client/models/identifier_request.rb b/lib/ontologies_api_client/models/identifier_request.rb new file mode 100644 index 0000000..9a1adac --- /dev/null +++ b/lib/ontologies_api_client/models/identifier_request.rb @@ -0,0 +1,13 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class IdentifierRequest < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + @media_type = "http://data.bioontology.org/metadata/IdentifierRequest" + end + end + end +end diff --git a/lib/ontologies_api_client/read_write.rb b/lib/ontologies_api_client/read_write.rb index 9aaa818..3eecd24 100644 --- a/lib/ontologies_api_client/read_write.rb +++ b/lib/ontologies_api_client/read_write.rb @@ -1,21 +1,38 @@ require 'active_support/core_ext/hash' +require_relative 'utils' module LinkedData module Client module ReadWrite HTTP = LinkedData::Client::HTTP - def save - resp = HTTP.post(self.class.collection_path, self.to_hash) - invalidate_cache() - resp + def save + begin + # LOGGER.debug("\n\n\n---------- ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite -> save : \n\n - self.class.collection_path:#{self.class.collection_path} \n\n - self.to_hash: #{self.to_hash}") + self.class.collection_path + resp = HTTP.post(self.class.collection_path, self.to_hash) + # LOGGER.debug("\n\n ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite -> save : \n\n - resp -> #{resp}") + invalidate_cache() + resp + rescue => e + LOGGER.debug("\n\nECCEZIONE! ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite - save() #{e.message}\n#{e.backtrace.join("\n")}") + raise e + end end def update(options = {}) - values = options[:values] || changed_values() - return if values.empty? - resp = HTTP.patch(self.id, values) - invalidate_cache() + begin + # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite -> update options:#{options.inspect}") + values = options[:values] || changed_values() + # LOGGER.debug("\n\n > values=#{values.inspect}") + return if values.empty? + # LOGGER.debug("\n\n > BEFORE CALL HTTP.patch") + resp = HTTP.patch(self.id, values) + invalidate_cache() + rescue => e + LOGGER.debug("\n\n ECCEZIONE! ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite: #{e.message}\n#{e.backtrace.join("\n")}") + raise e + end resp end @@ -24,6 +41,9 @@ def update_from_params(params) # service so we know that what we're updating is # actually the full object all_values = HTTP.get(self.id, include: "all") + + # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite -> update_from_params(params) :\n\n params=#{params.inspect}\n\n all_values=#{all_values.inspect}") + all_values.instance_variables.each do |var| self.send("#{var}=", all_values.instance_variable_get(var)) end @@ -36,14 +56,22 @@ def update_from_params(params) end def changed_values - existing = HTTP.get(self.id, include: "all") - changed_attrs = {} - self.instance_variables.each do |var| - var_sym = var[1..-1].to_sym - next if [:id, :type, :links, :context, :created].include?(var_sym) - new_value = self.instance_variable_get(var) - current_value = existing.instance_variable_get(var) - changed_attrs[var_sym] = new_value unless equivalent?(current_value, new_value) + begin + existing = HTTP.get(self.id, include: "all") + changed_attrs = {} + # LOGGER.debug("\n\n ================================\nONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite -> changed_values:\n\n > existing=#{existing.inspect}\n\n > self.instance_variables=#{self.instance_variables.inspect}") + self.instance_variables.each do |var| + # LOGGER.debug(" => var=#{var} >>> CALL equivalent"); + var_sym = var[1..-1].to_sym + next if [:id, :type, :links, :context, :created].include?(var_sym) + new_value = self.instance_variable_get(var) + current_value = existing.instance_variable_get(var) + changed_attrs[var_sym] = new_value unless equivalent?(current_value, new_value) + end + # LOGGER.debug("\n\n => changed_attrs=#{changed_attrs.inspect}\n") + rescue => e + LOGGER.debug("\n\nECCEZIONE! ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::changed_values: #{e.message}\n#{e.backtrace.join("\n")}") + raise e end changed_attrs end @@ -56,30 +84,77 @@ def delete private + def equivalent?(current_value, new_value) - # If we're comparing an existing embedded object - # then use the id for comparison - if current_value.is_a?(LinkedData::Client::Base) - return current_value.id.eql?(new_value) - end + # LOGGER.debug("\n\n----------------------\nONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite -> equivalent:\n\n > current_value=#{current_value.inspect}\n\n > new_value=#{new_value.inspect}") + begin + # If we're comparing an existing embedded object + # then use the id for comparison + if current_value.is_a?(LinkedData::Client::Base) + #LOGGER.debug(" > current_value is a LinkedData::Client::Base \n > current_value= #{current_value}\n > new_value=#{new_value}") + result_equivalent = current_value.id.eql?(new_value) + #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") + return result_equivalent + end + + # Otherwise, do some complex comparing + case new_value + when String + #LOGGER.debug(" > new_value is a String\n > current_value= #{current_value}\n > new_value=#{new_value}") + result_equivalent = current_value.to_s.eql?(new_value) + #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") + return result_equivalent + when Array, Hash + new_value = nil if new_value.is_a?(Array) && new_value.empty? && current_value.nil? + if new_value.is_a?(Hash) || (new_value.is_a?(Array) && (new_value.first.is_a?(Hash) || new_value.first.is_a?(OpenStruct))) + #LOGGER.debug(" > new_value is an Array && new_value.first is an Hash or OpenStruct") + + ordered_current = LinkedData::Client::Utils.order_inner_array_elements(current_value, true, [:id, :type, :links, :context, :created]) + # LOGGER.debug(" > > ordered_current=#{ordered_current}") - # Otherwise, do some complex comparing - case new_value - when String - return current_value.to_s.eql?(new_value) - when Array, Hash - new_value = nil if new_value.is_a?(Array) && new_value.empty? && current_value.nil? - if new_value.is_a?(Array) && new_value.first.is_a?(Hash) - clean_current = current_value.map {|e| e.to_h.symbolize_keys.delete_if {|k,v| v.nil?}}.sort_by {|e| e.is_a?(Hash) ? e.values.sort : e} - clean_new = new_value.map {|e| e.symbolize_keys.delete_if {|k,v| v.nil?}}.sort_by {|e| e.is_a?(Hash) ? e.values.sort : e} - return clean_current.eql?(clean_new) rescue clean_current == clean_new - else - return current_value.sort.eql?(new_value.sort) rescue current_value == new_value + clean_current = LinkedData::Client::Utils.recursive_symbolize_keys(ordered_current, true, [:id, :type, :links, :context, :created]) + # LOGGER.debug(" > > clean_current=#{clean_current}") + + # LOGGER.debug(" > > BEFORE elaboration PROVE: new_value=#{new_value}") + # LOGGER.debug(" > > BEFORE elaboration PROVE: new_value.inspect= #{new_value.inspect}") + # LOGGER.debug(" > > BEFORE elaboration PROVE: new_value_symbolized: #{LinkedData::Client::Utils.recursive_symbolize_keys(new_value)}") + #new_value.map{|e| LOGGER.debug("new_value.map: \n e = #{e}\n e.to_h = #{e.to_h}")} + + #new_value[0].each{ |k,v| LOGGER.debug("new_value[0].each [k,v] = [#{k},#{v}] => #{new_value[0]}")} + + clean_new_to_order = LinkedData::Client::Utils.recursive_symbolize_keys(new_value, true, [:id, :type, :links, :context, :created]) + # LOGGER.debug(" > > clean_new_to_order=#{clean_new_to_order}") + + clean_new = LinkedData::Client::Utils.order_inner_array_elements(clean_new_to_order, true, [:id, :type, :links, :context, :created]) + #LOGGER.debug(" > > clean_new=#{clean_new}") + + result_equivalent = clean_current.eql?(clean_new) rescue clean_current == clean_new + # LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") + return result_equivalent + else + #LOGGER.debug(" > new_value is an Array OR Hash:\n > current_value= #{current_value}\n > new_value=#{new_value}") + result_equivalent = current_value.sort.eql?(new_value.sort) rescue current_value == new_value + #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") + return result_equivalent + end end + #LOGGER.debug(" > OTHERWISE CASE > current_value= #{current_value}\n > new_value=#{new_value}") + result_equivalent = current_value.sort.eql?(new_value.sort) rescue current_value == new_value + #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") + return result_equivalent + rescue => e + LOGGER.debug("\n\n ECCEZIONE! ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::equivalent: #{e.message}\n#{e.backtrace.join("\n")}") + raise e end - return current_value.eql?(new_value) rescue current_value == new_value end + # NEW method Ecoportal + # clean an annidate hash (hash with array, hash or array of hash) + def clean_annidate_hash(obj) + + end + + def invalidate_cache self.class.all(invalidate_cache: true) HTTP.get(self.id, invalidate_cache: true) if self.id diff --git a/lib/ontologies_api_client/utils.rb b/lib/ontologies_api_client/utils.rb new file mode 100644 index 0000000..9bc2b53 --- /dev/null +++ b/lib/ontologies_api_client/utils.rb @@ -0,0 +1,101 @@ +# --- NEW ECOPORTAL class----- +# Utility class +module LinkedData + module Client + class Utils + + #Symbolize the keys recursively for the object passed as parameter + # Es. {"a" => "aaa", "b" => {"b1" => "bbb1", "b2" => "bbb2"}} + # becomes {:a => "aaa", :b => {:b1 => "bbb1", :b2 => "bbb2"}} + def self.recursive_symbolize_keys(obj, exclude_nil_value = false, excluded_keys = []) + # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::Utils -> recursive_symbolize_keys CALL:") + if obj.is_a?Array + ## LOGGER.debug(" OBJ is an Array") + obj.map do |e| + val = (e.to_h == {}) ? e : e.to_h + recursive_symbolize_keys(val, exclude_nil_value, excluded_keys) + end + else + {}.tap do |h| + obj.each do |key, value| + # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::Utils -> recursive_symbolize_keys [key,value] = [#{key}, #{value}] - key.to_sym= #{key.to_sym}") + # next if excluded_keys.include?(key) + # next if (exclude_nil_value && value.nil?) + h[key.to_sym] = map_value(value, exclude_nil_value, excluded_keys) + end + end + end + end + + def self.map_value(thing, exclude_nil_value = false, excluded_keys = []) + # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::Utils -> map_value: \n > thing = #{thing}") + case thing + when Hash + # LOGGER.debug(" > thing is HASH") + recursive_symbolize_keys(thing, exclude_nil_value, excluded_keys) + when Array + # LOGGER.debug(" > thing is Array") + thing.map { |v| map_value(v, exclude_nil_value, excluded_keys) } + when OpenStruct + # LOGGER.debug(" > thing is OpenStruct") + recursive_symbolize_keys(thing.to_h, exclude_nil_value, excluded_keys) + # when Base + # recursive_symbolize_keys(thing.to_hash, exclude_nil_value) + else + # LOGGER.debug(" > thing type is not recognized") + thing + end + end + + # UTILITY: be an object comparables with another one, indipendently the order of its fields or the order of + # its array elements + # For each array in the object (Hash or OpenStruct) or nested into subobject, this method order the elements. + # That permits to compare two objects (Hash or OpenStruct). + # Es. + # {:a => [3,1,2], :b => {:b1 => [30,10,50], :b2 => [300,100,500]}} + # becomes {:a => [1,2,3], :b => {:b1 => [10,30,50], :b2 => [100,300,500]}} + # + # In case the elements of array are hashs, then it orders them by their hash value + def self.order_inner_array_elements(obj, exclude_nil_value = false, excluded_keys = []) + if obj.is_a?Array + obj.map do |e| + val = (e.to_h == {}) ? e : e.to_h + order_inner_array_elements(val, exclude_nil_value, excluded_keys) + end + #obj.map{|e| order_inner_array_elements(e.to_h, exclude_nil_value, excluded_keys)} + else + {}.tap do |h| + obj.each do |key, value| + next if excluded_keys.include?(key) + next if (exclude_nil_value && value.nil?) + v = map_ordered_value(value, exclude_nil_value, excluded_keys) + h[key.to_sym] = v + end + end + end + end + + def self.map_ordered_value(thing, exclude_nil_value = false, excluded_keys = []) + case thing + when Hash + result = order_inner_array_elements(thing, exclude_nil_value, excluded_keys) + when Array + result = thing.map do |v| + case v + when Hash, Array, OpenStruct + map_ordered_value(v, exclude_nil_value, excluded_keys) + else + v + end + end + result = result.sort_by{|e| e.is_a?(Hash) ? e.hash : e} + when OpenStruct + result = order_inner_array_elements(thing.to_h, exclude_nil_value, excluded_keys) + else + result = thing + end + result + end + end + end +end \ No newline at end of file From fb61d253421151435e1dfaba7c1a18bcde66cfd8 Mon Sep 17 00:00:00 2001 From: Claudio Date: Mon, 8 Jun 2020 13:34:29 +0200 Subject: [PATCH 3/9] update .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index dcf52e4..19faf3a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ config/config.rb *.gem .DS_Store +.vs/slnx.sqlite +.vs/VSWorkspaceState.json +.vs/ontologies_api_ruby_client/v15/.suo From 80f524bb1468101d212e7e69352afcec20b62b32 Mon Sep 17 00:00:00 2001 From: Manuel Fiorelli Date: Tue, 6 Sep 2022 03:52:07 +0200 Subject: [PATCH 4/9] Initial commit --- .gitignore | 4 +- lib/ontologies_api_client.rb | 1 + lib/ontologies_api_client/base.rb | 15 ++- lib/ontologies_api_client/http.rb | 28 ++-- .../middleware/faraday-user-apikey.rb | 14 +- lib/ontologies_api_client/models/category.rb | 2 + lib/ontologies_api_client/models/creator.rb | 12 ++ .../models/creator_identifier.rb | 12 ++ lib/ontologies_api_client/models/group.rb | 2 + .../models/identifier_request.rb | 13 ++ lib/ontologies_api_client/models/note.rb | 2 +- lib/ontologies_api_client/models/review.rb | 17 +++ lib/ontologies_api_client/read_write.rb | 126 +++++++++++++----- lib/ontologies_api_client/utils.rb | 101 ++++++++++++++ 14 files changed, 297 insertions(+), 52 deletions(-) create mode 100644 lib/ontologies_api_client/models/creator.rb create mode 100644 lib/ontologies_api_client/models/creator_identifier.rb create mode 100644 lib/ontologies_api_client/models/identifier_request.rb create mode 100644 lib/ontologies_api_client/utils.rb diff --git a/.gitignore b/.gitignore index 9c6de78..8c37568 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ config/config.rb .idea/ # Specific to rbenv -.ruby-version \ No newline at end of file +.ruby-version + +./vs \ No newline at end of file diff --git a/lib/ontologies_api_client.rb b/lib/ontologies_api_client.rb index 739639a..8cce87b 100644 --- a/lib/ontologies_api_client.rb +++ b/lib/ontologies_api_client.rb @@ -2,6 +2,7 @@ require 'multi_json' require 'spawnling' +require_relative 'ontologies_api_client/utils' require_relative 'ontologies_api_client/config' require_relative 'ontologies_api_client/http' require_relative 'ontologies_api_client/link_explorer' diff --git a/lib/ontologies_api_client/base.rb b/lib/ontologies_api_client/base.rb index d764d7e..24aff58 100644 --- a/lib/ontologies_api_client/base.rb +++ b/lib/ontologies_api_client/base.rb @@ -39,12 +39,17 @@ def self.map_classes end def initialize(options = {}) - values = options[:values] - if values.is_a?(Hash) && !values.empty? - create_attributes(values.keys) - populate_attributes(values) + begin + values = options[:values] + if values.is_a?(Hash) && !values.empty? + create_attributes(values.keys) + populate_attributes(values) + end + create_attributes(self.class.attrs_always_present || []) + rescue => e + LOGGER.debug("\n\n ECCEZIONE! RUBY API - base.rb - initialize(options): #{e.message}\n#{e.backtrace.join("\n")}") + raise e end - create_attributes(self.class.attrs_always_present || []) end def id diff --git a/lib/ontologies_api_client/http.rb b/lib/ontologies_api_client/http.rb index ef57f9d..4bb9eb9 100644 --- a/lib/ontologies_api_client/http.rb +++ b/lib/ontologies_api_client/http.rb @@ -59,6 +59,8 @@ def self.get(path, params = {}, options = {}) begin puts "Getting: #{path} with #{params}" if $DEBUG + + # LOGGER.debug("\n\n RUBY API - LinkedData::Client::HTTP ->Get: #{path} with:\n > params:#{params.inspect}\n > options:#{options.inspect} ") begin response = conn.get do |req| req.url path @@ -103,17 +105,23 @@ def self.get_batch(paths, params = {}) end def self.post(path, obj, options = {}) - file, file_attribute = params_file_handler(obj) - response = conn.post do |req| - req.url path - custom_req(obj, file, file_attribute, req) - end - raise Exception, response.body if response.status >= 500 - if options[:raw] || false # return the unparsed body of the request - return response.body - else - return recursive_struct(load_json(response.body)) + begin + file, file_attribute = params_file_handler(obj) + response = conn.post do |req| + req.url path + custom_req(obj, file, file_attribute, req) + end + raise Exception, response.body if response.status >= 500 + if options[:raw] || false # return the unparsed body of the request + return response.body + else + return recursive_struct(load_json(response.body)) + end + rescue => e + LOGGER.debug "\n\n\nRUBY API - LinkedData::Client::HTTP ->post: - ECCEZIONE: #{e.message}\n#{e.backtrace.join("\n")}" + raise e end + end def self.put(path, obj) diff --git a/lib/ontologies_api_client/middleware/faraday-user-apikey.rb b/lib/ontologies_api_client/middleware/faraday-user-apikey.rb index 08c73cc..f9a6425 100644 --- a/lib/ontologies_api_client/middleware/faraday-user-apikey.rb +++ b/lib/ontologies_api_client/middleware/faraday-user-apikey.rb @@ -7,11 +7,19 @@ def initialize(app, *arguments) end def call(env) - user = Thread.current[:session] && Thread.current[:session][:user] ? Thread.current[:session][:user] : nil - if user + bearer_token = Thread.current[:ontologies_api_client_token] + + if bearer_token headers = env[:request_headers] - headers["Authorization"] = headers["Authorization"] + "&userapikey=" + user.apikey + headers["Authorization"] = "Bearer #{bearer_token}" env[:request_headers] = headers + else + user = Thread.current[:session] && Thread.current[:session][:user] ? Thread.current[:session][:user] : nil + if user + headers = env[:request_headers] + headers["Authorization"] = headers["Authorization"] + "&userapikey=" + user.apikey + env[:request_headers] = headers + end end @app.call(env) end diff --git a/lib/ontologies_api_client/models/category.rb b/lib/ontologies_api_client/models/category.rb index 0f41b59..7df0645 100644 --- a/lib/ontologies_api_client/models/category.rb +++ b/lib/ontologies_api_client/models/category.rb @@ -5,6 +5,8 @@ module Client module Models class Category < LinkedData::Client::Base include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + @media_type = "http://data.bioontology.org/metadata/Category" end end diff --git a/lib/ontologies_api_client/models/creator.rb b/lib/ontologies_api_client/models/creator.rb new file mode 100644 index 0000000..4a9569a --- /dev/null +++ b/lib/ontologies_api_client/models/creator.rb @@ -0,0 +1,12 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class Creator < LinkedData::Client::Base + include LinkedData::Client::ReadWrite + @media_type = "http://data.ecoportal.org/metadata/Creator" + end + end + end +end diff --git a/lib/ontologies_api_client/models/creator_identifier.rb b/lib/ontologies_api_client/models/creator_identifier.rb new file mode 100644 index 0000000..c7da7c8 --- /dev/null +++ b/lib/ontologies_api_client/models/creator_identifier.rb @@ -0,0 +1,12 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class CreatorIdentifier < LinkedData::Client::Base + include LinkedData::Client::ReadWrite + @media_type = "http://data.ecoportal.org/metadata/CreatorIdentifier" + end + end + end +end diff --git a/lib/ontologies_api_client/models/group.rb b/lib/ontologies_api_client/models/group.rb index 803b148..3061d72 100644 --- a/lib/ontologies_api_client/models/group.rb +++ b/lib/ontologies_api_client/models/group.rb @@ -5,6 +5,8 @@ module Client module Models class Group < LinkedData::Client::Base include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + @media_type = "http://data.bioontology.org/metadata/Group" end end diff --git a/lib/ontologies_api_client/models/identifier_request.rb b/lib/ontologies_api_client/models/identifier_request.rb new file mode 100644 index 0000000..9a1adac --- /dev/null +++ b/lib/ontologies_api_client/models/identifier_request.rb @@ -0,0 +1,13 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class IdentifierRequest < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + @media_type = "http://data.bioontology.org/metadata/IdentifierRequest" + end + end + end +end diff --git a/lib/ontologies_api_client/models/note.rb b/lib/ontologies_api_client/models/note.rb index a24c534..24b15b3 100644 --- a/lib/ontologies_api_client/models/note.rb +++ b/lib/ontologies_api_client/models/note.rb @@ -9,7 +9,7 @@ class Note < LinkedData::Client::Base @media_type = "http://data.bioontology.org/metadata/Note" def deletable?(user) - false + !user.nil? && user.admin? # Administrators can delete notes end def uuid diff --git a/lib/ontologies_api_client/models/review.rb b/lib/ontologies_api_client/models/review.rb index bd7cae5..c72cfe8 100644 --- a/lib/ontologies_api_client/models/review.rb +++ b/lib/ontologies_api_client/models/review.rb @@ -8,6 +8,23 @@ class Review < LinkedData::Client::Base include LinkedData::Client::ReadWrite @media_type = "http://data.bioontology.org/metadata/Review" + @include_attrs = "all" + + def self.more_recent_review(r1, r2) + time1 = r1.created + time2 = r2.created + + time2 <=> time1 # reversed order + end + + def self.sort_reviews(reviews) + reviews.sort { |r1, r2| more_recent_review(r1, r2) } + end + + def self.sort_reviews!(reviews) + reviews.sort! { |r1, r2| more_recent_review(r1, r2) } + reviews + end end end end diff --git a/lib/ontologies_api_client/read_write.rb b/lib/ontologies_api_client/read_write.rb index ff2461e..e3281bc 100644 --- a/lib/ontologies_api_client/read_write.rb +++ b/lib/ontologies_api_client/read_write.rb @@ -1,24 +1,35 @@ require 'active_support/core_ext/hash' +require_relative 'utils' module LinkedData module Client module ReadWrite HTTP = LinkedData::Client::HTTP - def save(options = {}) - resp = HTTP.post(self.class.collection_path, self.to_hash) + def save(options = {}) + begin + resp = HTTP.post(self.class.collection_path, self.to_hash) + # LOGGER.debug("\n\n ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite -> save : \n\n - resp -> #{resp}") # cache_refresh_all allow to avoid to refresh everything, to make it faster when saving new submission invalidate_cache(options[:cache_refresh_all] == false) resp + rescue => e + LOGGER.debug("\n\nECCEZIONE! ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite - save() #{e.message}\n#{e.backtrace.join("\n")}") + raise e + end end def update(options = {}) - values = options[:values] || changed_values() - return if values.empty? - resp = HTTP.patch(self.id, values) + begin + values = options[:values] || changed_values() + return if values.empty? + resp = HTTP.patch(self.id, values) # When updating submission we avoid refreshing all cache to avoid calling /submissions?display=all that takes a lot of time invalidate_cache(options[:cache_refresh_all] == false) - + rescue => e + LOGGER.debug("\n\n ECCEZIONE! ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite: #{e.message}\n#{e.backtrace.join("\n")}") + raise e + end resp end @@ -39,14 +50,19 @@ def update_from_params(params) end def changed_values - existing = HTTP.get(self.id, include: "all") - changed_attrs = {} - self.instance_variables.each do |var| - var_sym = var[1..-1].to_sym - next if [:id, :type, :links, :context, :created].include?(var_sym) - new_value = self.instance_variable_get(var) - current_value = existing.instance_variable_get(var) - changed_attrs[var_sym] = new_value unless equivalent?(current_value, new_value) + begin + existing = HTTP.get(self.id, include: "all") + changed_attrs = {} + self.instance_variables.each do |var| + var_sym = var[1..-1].to_sym + next if [:id, :type, :links, :context, :created].include?(var_sym) + new_value = self.instance_variable_get(var) + current_value = existing.instance_variable_get(var) + changed_attrs[var_sym] = new_value unless equivalent?(current_value, new_value) + end + rescue => e + LOGGER.debug("\n\nECCEZIONE! ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::changed_values: #{e.message}\n#{e.backtrace.join("\n")}") + raise e end changed_attrs end @@ -60,29 +76,75 @@ def delete private def equivalent?(current_value, new_value) - # If we're comparing an existing embedded object - # then use the id for comparison - if current_value.is_a?(LinkedData::Client::Base) - return current_value.id.eql?(new_value) - end + # LOGGER.debug("\n\n----------------------\nONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite -> equivalent:\n\n > current_value=#{current_value.inspect}\n\n > new_value=#{new_value.inspect}") + begin + # If we're comparing an existing embedded object + # then use the id for comparison + if current_value.is_a?(LinkedData::Client::Base) + #LOGGER.debug(" > current_value is a LinkedData::Client::Base \n > current_value= #{current_value}\n > new_value=#{new_value}") + result_equivalent = current_value.id.eql?(new_value) + #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") + return result_equivalent + end - # Otherwise, do some complex comparing - case new_value - when String - return current_value.to_s.eql?(new_value) - when Array, Hash - new_value = nil if new_value.is_a?(Array) && new_value.empty? && current_value.nil? - if new_value.is_a?(Array) && new_value.first.is_a?(Hash) - clean_current = current_value.map {|e| e.to_h.symbolize_keys.delete_if {|k,v| v.nil?}}.sort_by {|e| e.is_a?(Hash) ? e.values.sort : e} - clean_new = new_value.map {|e| e.symbolize_keys.delete_if {|k,v| v.nil?}}.sort_by {|e| e.is_a?(Hash) ? e.values.sort : e} - return clean_current.eql?(clean_new) rescue clean_current == clean_new - else - return current_value.sort.eql?(new_value.sort) rescue current_value == new_value + # Otherwise, do some complex comparing + case new_value + when String + #LOGGER.debug(" > new_value is a String\n > current_value= #{current_value}\n > new_value=#{new_value}") + result_equivalent = current_value.to_s.eql?(new_value) + #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") + return result_equivalent + when Array, Hash + new_value = nil if new_value.is_a?(Array) && new_value.empty? && current_value.nil? + if new_value.is_a?(Hash) || (new_value.is_a?(Array) && (new_value.first.is_a?(Hash) || new_value.first.is_a?(OpenStruct))) + #LOGGER.debug(" > new_value is an Array && new_value.first is an Hash or OpenStruct") + + ordered_current = LinkedData::Client::Utils.order_inner_array_elements(current_value, true, [:id, :type, :links, :context, :created]) + # LOGGER.debug(" > > ordered_current=#{ordered_current}") + + clean_current = LinkedData::Client::Utils.recursive_symbolize_keys(ordered_current, true, [:id, :type, :links, :context, :created]) + # LOGGER.debug(" > > clean_current=#{clean_current}") + + # LOGGER.debug(" > > BEFORE elaboration PROVE: new_value=#{new_value}") + # LOGGER.debug(" > > BEFORE elaboration PROVE: new_value.inspect= #{new_value.inspect}") + # LOGGER.debug(" > > BEFORE elaboration PROVE: new_value_symbolized: #{LinkedData::Client::Utils.recursive_symbolize_keys(new_value)}") + #new_value.map{|e| LOGGER.debug("new_value.map: \n e = #{e}\n e.to_h = #{e.to_h}")} + + #new_value[0].each{ |k,v| LOGGER.debug("new_value[0].each [k,v] = [#{k},#{v}] => #{new_value[0]}")} + + clean_new_to_order = LinkedData::Client::Utils.recursive_symbolize_keys(new_value, true, [:id, :type, :links, :context, :created]) + # LOGGER.debug(" > > clean_new_to_order=#{clean_new_to_order}") + + clean_new = LinkedData::Client::Utils.order_inner_array_elements(clean_new_to_order, true, [:id, :type, :links, :context, :created]) + #LOGGER.debug(" > > clean_new=#{clean_new}") + + result_equivalent = clean_current.eql?(clean_new) rescue clean_current == clean_new + # LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") + return result_equivalent + else + #LOGGER.debug(" > new_value is an Array OR Hash:\n > current_value= #{current_value}\n > new_value=#{new_value}") + result_equivalent = current_value.sort.eql?(new_value.sort) rescue current_value == new_value + #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") + return result_equivalent + end end + #LOGGER.debug(" > OTHERWISE CASE > current_value= #{current_value}\n > new_value=#{new_value}") + result_equivalent = current_value.sort.eql?(new_value.sort) rescue current_value == new_value + #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") + return result_equivalent + rescue => e + LOGGER.debug("\n\n ECCEZIONE! ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::equivalent: #{e.message}\n#{e.backtrace.join("\n")}") + raise e end - return current_value.eql?(new_value) rescue current_value == new_value end + # NEW method Ecoportal + # clean an annidate hash (hash with array, hash or array of hash) + def clean_annidate_hash(obj) + + end + + def invalidate_cache(cache_refresh_all = true) self.class.all(invalidate_cache: true) if cache_refresh_all HTTP.get(self.id, invalidate_cache: true) if self.id diff --git a/lib/ontologies_api_client/utils.rb b/lib/ontologies_api_client/utils.rb new file mode 100644 index 0000000..9bc2b53 --- /dev/null +++ b/lib/ontologies_api_client/utils.rb @@ -0,0 +1,101 @@ +# --- NEW ECOPORTAL class----- +# Utility class +module LinkedData + module Client + class Utils + + #Symbolize the keys recursively for the object passed as parameter + # Es. {"a" => "aaa", "b" => {"b1" => "bbb1", "b2" => "bbb2"}} + # becomes {:a => "aaa", :b => {:b1 => "bbb1", :b2 => "bbb2"}} + def self.recursive_symbolize_keys(obj, exclude_nil_value = false, excluded_keys = []) + # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::Utils -> recursive_symbolize_keys CALL:") + if obj.is_a?Array + ## LOGGER.debug(" OBJ is an Array") + obj.map do |e| + val = (e.to_h == {}) ? e : e.to_h + recursive_symbolize_keys(val, exclude_nil_value, excluded_keys) + end + else + {}.tap do |h| + obj.each do |key, value| + # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::Utils -> recursive_symbolize_keys [key,value] = [#{key}, #{value}] - key.to_sym= #{key.to_sym}") + # next if excluded_keys.include?(key) + # next if (exclude_nil_value && value.nil?) + h[key.to_sym] = map_value(value, exclude_nil_value, excluded_keys) + end + end + end + end + + def self.map_value(thing, exclude_nil_value = false, excluded_keys = []) + # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::Utils -> map_value: \n > thing = #{thing}") + case thing + when Hash + # LOGGER.debug(" > thing is HASH") + recursive_symbolize_keys(thing, exclude_nil_value, excluded_keys) + when Array + # LOGGER.debug(" > thing is Array") + thing.map { |v| map_value(v, exclude_nil_value, excluded_keys) } + when OpenStruct + # LOGGER.debug(" > thing is OpenStruct") + recursive_symbolize_keys(thing.to_h, exclude_nil_value, excluded_keys) + # when Base + # recursive_symbolize_keys(thing.to_hash, exclude_nil_value) + else + # LOGGER.debug(" > thing type is not recognized") + thing + end + end + + # UTILITY: be an object comparables with another one, indipendently the order of its fields or the order of + # its array elements + # For each array in the object (Hash or OpenStruct) or nested into subobject, this method order the elements. + # That permits to compare two objects (Hash or OpenStruct). + # Es. + # {:a => [3,1,2], :b => {:b1 => [30,10,50], :b2 => [300,100,500]}} + # becomes {:a => [1,2,3], :b => {:b1 => [10,30,50], :b2 => [100,300,500]}} + # + # In case the elements of array are hashs, then it orders them by their hash value + def self.order_inner_array_elements(obj, exclude_nil_value = false, excluded_keys = []) + if obj.is_a?Array + obj.map do |e| + val = (e.to_h == {}) ? e : e.to_h + order_inner_array_elements(val, exclude_nil_value, excluded_keys) + end + #obj.map{|e| order_inner_array_elements(e.to_h, exclude_nil_value, excluded_keys)} + else + {}.tap do |h| + obj.each do |key, value| + next if excluded_keys.include?(key) + next if (exclude_nil_value && value.nil?) + v = map_ordered_value(value, exclude_nil_value, excluded_keys) + h[key.to_sym] = v + end + end + end + end + + def self.map_ordered_value(thing, exclude_nil_value = false, excluded_keys = []) + case thing + when Hash + result = order_inner_array_elements(thing, exclude_nil_value, excluded_keys) + when Array + result = thing.map do |v| + case v + when Hash, Array, OpenStruct + map_ordered_value(v, exclude_nil_value, excluded_keys) + else + v + end + end + result = result.sort_by{|e| e.is_a?(Hash) ? e.hash : e} + when OpenStruct + result = order_inner_array_elements(thing.to_h, exclude_nil_value, excluded_keys) + else + result = thing + end + result + end + end + end +end \ No newline at end of file From 89c6d9895a9767731d562804d2419da761d1bc6c Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Sat, 15 Apr 2023 13:42:54 +0100 Subject: [PATCH 5/9] add identifier_request model --- .../models/identifier_request.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 lib/ontologies_api_client/models/identifier_request.rb diff --git a/lib/ontologies_api_client/models/identifier_request.rb b/lib/ontologies_api_client/models/identifier_request.rb new file mode 100644 index 0000000..9a1adac --- /dev/null +++ b/lib/ontologies_api_client/models/identifier_request.rb @@ -0,0 +1,13 @@ +require_relative "../base" + +module LinkedData + module Client + module Models + class IdentifierRequest < LinkedData::Client::Base + include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + @media_type = "http://data.bioontology.org/metadata/IdentifierRequest" + end + end + end +end From e932948101d37cebbeb7493db5495e09467a0058 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Sat, 15 Apr 2023 13:46:37 +0100 Subject: [PATCH 6/9] add the possibility to use bearer_token for authentification --- .../middleware/faraday-user-apikey.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/ontologies_api_client/middleware/faraday-user-apikey.rb b/lib/ontologies_api_client/middleware/faraday-user-apikey.rb index 08c73cc..f9a6425 100644 --- a/lib/ontologies_api_client/middleware/faraday-user-apikey.rb +++ b/lib/ontologies_api_client/middleware/faraday-user-apikey.rb @@ -7,11 +7,19 @@ def initialize(app, *arguments) end def call(env) - user = Thread.current[:session] && Thread.current[:session][:user] ? Thread.current[:session][:user] : nil - if user + bearer_token = Thread.current[:ontologies_api_client_token] + + if bearer_token headers = env[:request_headers] - headers["Authorization"] = headers["Authorization"] + "&userapikey=" + user.apikey + headers["Authorization"] = "Bearer #{bearer_token}" env[:request_headers] = headers + else + user = Thread.current[:session] && Thread.current[:session][:user] ? Thread.current[:session][:user] : nil + if user + headers = env[:request_headers] + headers["Authorization"] = headers["Authorization"] + "&userapikey=" + user.apikey + env[:request_headers] = headers + end end @app.call(env) end From fc70caa18720c3e64b8f541eec1dd228cabead2e Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Sat, 15 Apr 2023 13:41:30 +0100 Subject: [PATCH 7/9] enable read/write feature to Group and Category models --- lib/ontologies_api_client/models/category.rb | 2 ++ lib/ontologies_api_client/models/group.rb | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/ontologies_api_client/models/category.rb b/lib/ontologies_api_client/models/category.rb index 0f41b59..7df0645 100644 --- a/lib/ontologies_api_client/models/category.rb +++ b/lib/ontologies_api_client/models/category.rb @@ -5,6 +5,8 @@ module Client module Models class Category < LinkedData::Client::Base include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + @media_type = "http://data.bioontology.org/metadata/Category" end end diff --git a/lib/ontologies_api_client/models/group.rb b/lib/ontologies_api_client/models/group.rb index 803b148..3061d72 100644 --- a/lib/ontologies_api_client/models/group.rb +++ b/lib/ontologies_api_client/models/group.rb @@ -5,6 +5,8 @@ module Client module Models class Group < LinkedData::Client::Base include LinkedData::Client::Collection + include LinkedData::Client::ReadWrite + @media_type = "http://data.bioontology.org/metadata/Group" end end From 24fb2549f7b69841e052491439bc8375ed5acfd9 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 27 May 2024 17:08:04 +0200 Subject: [PATCH 8/9] Merge to master: Release 2.1.0 - Refactor the cache code (#16) * use $DEBUG_API_CLIENT instead of $DEBUG for debugging request * add $API_CLIENT_INVALIDATE_CACHE option to disable cache * save and re-use the top_level_links http response as the result do not change * deprecate the find method in favor of get method for better performance * disable the refresh_cache on each update call * Refactor: update cache code and add tests (#15) * add tests for the api cache code * refactor the cache middleware code * remove unused function in the cache middleware * fix refactored cache code not getting the ld_obj when updating the cache --- Gemfile | 3 +- Gemfile.lock | 14 ++ config/config.test.rb | 9 +- lib/ontologies_api_client/collection.rb | 2 +- .../middleware/faraday-object-cache.rb | 213 ++++++++---------- test/middleware/test_cache.rb | 141 ++++++++++++ 6 files changed, 263 insertions(+), 119 deletions(-) create mode 100644 test/middleware/test_cache.rb diff --git a/Gemfile b/Gemfile index acdf917..b7f6653 100644 --- a/Gemfile +++ b/Gemfile @@ -4,4 +4,5 @@ gemspec gem 'rake' gem 'pry' -gem 'test-unit' \ No newline at end of file +gem 'test-unit' +gem 'webmock', require: false \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index 77dace9..413fd8c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -20,8 +20,14 @@ GEM i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) + addressable (2.8.6) + public_suffix (>= 2.0.2, < 6.0) + bigdecimal (3.1.7) coderay (1.1.3) concurrent-ruby (1.1.10) + crack (1.0.0) + bigdecimal + rexml excon (0.95.0) faraday (2.0.1) faraday-net_http (~> 2.0) @@ -32,6 +38,7 @@ GEM faraday-multipart (1.0.4) multipart-post (~> 2) faraday-net_http (2.1.0) + hashdiff (1.1.0) i18n (1.12.0) concurrent-ruby (~> 1.0) lz4-ruby (0.3.3) @@ -44,13 +51,19 @@ GEM pry (0.14.1) coderay (~> 1.1) method_source (~> 1.0) + public_suffix (5.0.5) rake (13.0.6) + rexml (3.2.6) ruby2_keywords (0.0.5) spawnling (2.1.5) test-unit (3.5.7) power_assert tzinfo (2.0.5) concurrent-ruby (~> 1.0) + webmock (3.23.0) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) PLATFORMS x86_64-darwin-21 @@ -62,6 +75,7 @@ DEPENDENCIES pry rake test-unit + webmock BUNDLED WITH 2.4.21 diff --git a/config/config.test.rb b/config/config.test.rb index af6837b..9185c5d 100644 --- a/config/config.test.rb +++ b/config/config.test.rb @@ -3,11 +3,12 @@ # be set via ENV variable UT_APIKEY abort('UT_APIKEY env variable is not set. Canceling tests') unless ENV.include?('UT_APIKEY') abort('UT_APIKEY env variable is set to an empty value. Canceling tests') unless ENV['UT_APIKEY'].size > 5 - +$API_CLIENT_INVALIDATE_CACHE = false +$DEBUG_API_CLIENT = false LinkedData::Client.config do |config| config.rest_url = 'https://data.bioontology.org' - config.apikey = ENV['UT_APIKEY'] -# config.apikey = 'xxxxx-xxxxx-xxxxxxxxxx' + config.apikey = '8b5b7825-538d-40e0-9e9e-5ab9274a9aeb' config.links_attr = 'links' - config.cache = false + config.cache = true + config.debug_client = false end diff --git a/lib/ontologies_api_client/collection.rb b/lib/ontologies_api_client/collection.rb index 6e0e36f..77572bf 100644 --- a/lib/ontologies_api_client/collection.rb +++ b/lib/ontologies_api_client/collection.rb @@ -77,7 +77,7 @@ def where(params = {}, &block) # Find a resource by id # @deprecated replace with "get" def find(id, params = {}) - [get(id, params)] + get(id, params) end ## diff --git a/lib/ontologies_api_client/middleware/faraday-object-cache.rb b/lib/ontologies_api_client/middleware/faraday-object-cache.rb index f5e06d3..080c416 100644 --- a/lib/ontologies_api_client/middleware/faraday-object-cache.rb +++ b/lib/ontologies_api_client/middleware/faraday-object-cache.rb @@ -8,107 +8,47 @@ class ObjectCacheResponse < Faraday::Response attr_accessor :parsed_body end - ## - # This middleware causes Faraday to return an actual object instead of a response - # This is done so that the object is cached instead of the unparsed json body. - # Otherwise, we have to re-parse the json on every cache hit, which is extrememly costly - # when compared to unmarshaling an object. class ObjectCache < Faraday::Middleware def initialize(app, *arguments) super(app) + options = arguments.last.is_a?(Hash) ? arguments.pop : {} + @logger = options.delete(:logger) + @store = options[:store] || ActiveSupport::Cache.lookup_store(nil, options) + end - if arguments.last.is_a? Hash - options = arguments.pop - @logger = options.delete(:logger) - else - options = arguments - end - @store = options[:store] || ActiveSupport::Cache.lookup_store(store, options) + def last_modified_key_id(request_key) + "LM::#{request_key}" + end + + def last_retrieved_key_id(request_key) + "LR::#{request_key}" end def call(env) invalidate_cache = env[:request_headers].delete(:invalidate_cache) - # Add if newer than last modified statement to headers request_key = cache_key_for(create_request(env)) - last_modified_key = "LM::#{request_key}" - last_retrieved_key = "LR::#{request_key}" + last_modified_key = last_modified_key_id(request_key) + last_retrieved_key = last_retrieved_key_id(request_key) - # If we invalidate the cache, then it forces a clean request if invalidate_cache - cache_delete(request_key) - cache_delete(last_modified_key) - cache_delete(last_retrieved_key) + delete_cache_entries(request_key, last_modified_key, last_retrieved_key) env[:request_headers]["Cache-Control"] = "no-cache" puts "Invalidated key #{request_key}" if enable_debug(request_key) end - # If we made the last request within the expiry if cache_exist?(last_retrieved_key) && cache_exist?(request_key) puts "Not expired: #{env[:url].to_s}, key #{request_key}" if enable_debug(request_key) - cached_item = cache_read(request_key) - ld_obj = cached_item.is_a?(Hash) && cached_item.key?(:ld_obj) ? cached_item[:ld_obj] : cached_item - env[:status] = 304 - cached_response = ObjectCacheResponse.new(env) - cached_response.parsed_body = ld_obj - return cached_response + return retrieve_cached_response(request_key) end - last_modified = cache_read(last_modified_key) headers = env[:request_headers] - puts "last modified: " + last_modified.to_s if last_modified && enable_debug(request_key) - headers['If-Modified-Since'] = last_modified if last_modified + headers['If-Modified-Since'] = cache_read(last_modified_key) if cache_read(last_modified_key) @app.call(env).on_complete do |response_env| - # Only cache get and head requests if [:get, :head].include?(response_env[:method]) - puts "Response status for key #{request_key}: " + response_env[:status].to_s if enable_debug(request_key) - last_modified = response_env[:response_headers]["Last-Modified"] - # Generate key using header hash - key = request_key - - # If the last retrieve time is less than expiry - if response_env[:status] == 304 && cache_exist?(key) - stored_obj = cache_read(key) - - # Update if last modified is different - if stored_obj[:last_modified] != last_modified - puts "Updating cache #{response_env[:url].to_s}, key #{request_key}" if enable_debug(request_key) - stored_obj[:last_modified] = last_modified - cache_write(last_modified_key, last_modified) - cache_write(key, stored_obj) - end - - ld_obj = stored_obj[:ld_obj] - else - if response_env[:body].nil? || response_env[:body].empty? - # We got here with an empty body, meaning the object wasn't - # in the cache (weird). So re-do the request. - puts "REDOING REQUEST, NO CACHE ENTRY for #{response_env[:url].to_s}, key #{request_key}" if enable_debug(request_key) - env[:request_headers].delete("If-Modified-Since") - response_env = @app.call(env).env - puts "REDOING REQUEST expiry: #{response_env[:response_headers]["Cache-Control"]}, last_modified: #{last_modified} for key #{request_key}" if enable_debug(request_key) - end - - ld_obj = LinkedData::Client::HTTP.object_from_json(response_env[:body]) - # This stmt was missing in the old code, resulting in repeated calls to REST because object failed to cache - last_modified = response_env[:response_headers]["Last-Modified"] - expiry = response_env[:response_headers]["Cache-Control"].to_s.split("max-age=").last.to_i - puts "Before storing object: expiry: #{expiry}, last_modified: #{last_modified} for key #{request_key}" if enable_debug(request_key) - - if expiry > 0 && last_modified - # This request is cacheable, store it - puts "Storing object: #{response_env[:url].to_s}, key #{request_key}" if enable_debug(request_key) - stored_obj = {last_modified: last_modified, ld_obj: ld_obj} - cache_write(last_modified_key, last_modified) - cache_write(last_retrieved_key, true, expires_in: expiry) - cache_write(key, stored_obj) - end - end - - response = ObjectCacheResponse.new(response_env) - response.parsed_body = ld_obj + response = process_response(response_env, request_key) return response end end @@ -117,8 +57,83 @@ def call(env) private def enable_debug(key) - return true if LinkedData::Client.settings.debug_client && (LinkedData::Client.settings.debug_client_keys.empty? || LinkedData::Client.settings.debug_client_keys.include?(key)) - false + LinkedData::Client.settings.debug_client && (LinkedData::Client.settings.debug_client_keys.empty? || LinkedData::Client.settings.debug_client_keys.include?(key)) + end + + def delete_cache_entries(*keys) + keys.each { |key| cache_delete(key) } + end + + def retrieve_cached_response(request_key) + cached_item = cache_read(request_key) + ld_obj = cached_item.is_a?(Hash) && cached_item.key?(:ld_obj) ? cached_item[:ld_obj] : cached_item + env = { status: 304 } + cached_response = ObjectCacheResponse.new(env) + cached_response.parsed_body = ld_obj + cached_response.env.response_headers = { "x-rack-cache" => 'hit' } + cached_response + end + + def process_response(response_env, request_key) + last_modified = response_env[:response_headers]["Last-Modified"] + key = request_key + cache_state = "miss" + + if response_env[:status] == 304 && cache_exist?(key) + cache_state = "fresh" + ld_obj = update_cache(request_key, last_modified) + else + ld_obj = cache_response(response_env, request_key) + end + + response = ObjectCacheResponse.new(response_env) + response.parsed_body = ld_obj + response.env.response_headers["x-rack-cache"] = cache_state + response + end + + def update_cache(request_key, last_modified) + stored_obj = cache_read(request_key) + if stored_obj[:last_modified] != last_modified + stored_obj[:last_modified] = last_modified + cache_write(last_modified_key_id(request_key), last_modified) + cache_write(request_key, stored_obj) + end + stored_obj.is_a?(Hash) && stored_obj.key?(:ld_obj) ? stored_obj[:ld_obj] : stored_obj + end + + def cache_response(response_env, request_key) + last_modified = response_env[:response_headers]["Last-Modified"] + + if response_env[:body].nil? || response_env[:body].empty? + # We got here with an empty body, meaning the object wasn't + # in the cache (weird). So re-do the request. + puts "REDOING REQUEST, NO CACHE ENTRY for #{response_env[:url].to_s}, key #{request_key}" if enable_debug(request_key) + response_env[:request_headers].delete("If-Modified-Since") + + response_env = @app.call(response_env).env + puts "REDOING REQUEST expiry: #{response_env[:response_headers]["Cache-Control"]}, last_modified: #{last_modified} for key #{request_key}" if enable_debug(request_key) + end + + + return nil if response_env[:body].nil? || response_env[:body].empty? + + ld_obj = LinkedData::Client::HTTP.object_from_json(response_env[:body]) + + expiry = response_env[:response_headers]["Cache-Control"].to_s.split("max-age=").last.to_i + + if expiry > 0 && last_modified + store_cache(request_key, ld_obj, last_modified, expiry) + end + + ld_obj + end + + def store_cache(request_key, ld_obj, last_modified, expiry) + stored_obj = { last_modified: last_modified, ld_obj: ld_obj } + cache_write(last_modified_key_id(request_key), last_modified) + cache_write(last_retrieved_key_id(request_key), true, expires_in: expiry) + cache_write(request_key, stored_obj) end def cache_write(key, obj, *args) @@ -132,12 +147,6 @@ def cache_write(key, obj, *args) if result return result else - # This should still get stored in memcache - # keep it in memory, though, because - # marshal/unmarshal is too slow. - # This way memcache will act as a backup - # and you load from there if it isn't - # in memory yet. @large_object_cache ||= {} @large_object_cache[key] = obj cache_write_compressed(key, obj, *args) @@ -147,11 +156,10 @@ def cache_write(key, obj, *args) def cache_read(key) obj = @store.read(key) - return if obj.nil? + return unless obj + if obj.is_a?(CompressedMemcache) - # Try to get from the large object cache large_obj = @large_object_cache[key] if @large_object_cache - # Fallback to the memcache version large_obj ||= cache_read_compressed(key) obj = large_obj end @@ -162,13 +170,14 @@ def cache_exist?(key) @store.exist?(key) end - class CompressedMemcache; attr_accessor :key; end + class CompressedMemcache + attr_accessor :key + end - ## - # Compress cache entry def cache_write_compressed(key, obj, *args) compressed = LZ4::compress(Marshal.dump(obj)) - return if compressed.nil? + return unless compressed + placeholder = CompressedMemcache.new placeholder.key = "#{key}::#{(Time.now.to_f * 1000).to_i}::LZ4" begin @@ -181,8 +190,6 @@ def cache_write_compressed(key, obj, *args) end end - ## - # Read compressed cache entry def cache_read_compressed(key) obj = @store.read(key) if obj.is_a?(CompressedMemcache) @@ -190,7 +197,6 @@ def cache_read_compressed(key) uncompressed = LZ4::uncompress(@store.read(obj.key)) obj = Marshal.load(uncompressed) rescue StandardError => e - # There is a problem with the stored value, let's remove it so we don't get the error again @store.delete(key) @store.delete(obj.key) raise e @@ -203,35 +209,16 @@ def cache_delete(key) @store.delete(key) end - # Internal: Generates a String key for a given request object. - # The request object is folded into a sorted Array (since we can't count - # on hashes order on Ruby 1.8), encoded as JSON and digested as a `SHA1` - # string. - # - # Returns the encoded String. def cache_key_for(request) array = request.stringify_keys.to_a.sort Digest::SHA1.hexdigest(Marshal.dump(array)) end - # Internal: Creates a new 'Hash' containing the request information. - # - # env - the environment 'Hash' from the Faraday stack. - # - # Returns a 'Hash' containing the ':method', ':url' and 'request_headers' - # entries. def create_request(env) request = env.to_hash.slice(:method, :url, :request_headers) request[:request_headers] = request[:request_headers].dup request end - - def clean_request_headers(request) - request[:request_headers].delete("If-Modified-Since") - request[:request_headers].delete("Expect") - request - end - end end diff --git a/test/middleware/test_cache.rb b/test/middleware/test_cache.rb new file mode 100644 index 0000000..ddea7a9 --- /dev/null +++ b/test/middleware/test_cache.rb @@ -0,0 +1,141 @@ +require_relative '../test_case' +require 'faraday' +require 'active_support' +require 'active_support/cache' +require_relative '../../lib/ontologies_api_client/middleware/faraday-object-cache' +require 'pry' +require 'benchmark' +require 'webmock' + +class FaradayObjectCacheTest < LinkedData::Client::TestCase + def setup + WebMock.disable! + apikey = LinkedData::Client.settings.apikey + @url = "#{LinkedData::Client.settings.rest_url}/ontologies/SNOMEDCT?apikey=#{apikey}" + + @cache_store = ActiveSupport::Cache::MemoryStore.new + @app = Faraday.new(url: @url) do |faraday| + faraday.use Faraday::ObjectCache, store: @cache_store + faraday.adapter :excon + end + end + + def test_cache_hit_for_get_request + body1, body2 = nil + # First request should not hit the cache + time1 = Benchmark.realtime do + response1 = @app.get + assert_equal 200, response1.status + assert uncached?(response1) + + body1 = JSON.parse(response1.body) + end + + time2 = Benchmark.realtime do + # Second request should hit the cache + response2 = @app.get + assert_equal 304, response2.status + assert cached?(response2) + body2 = response2.parsed_body.to_hash.stringify_keys + end + + assert time2 < time1 + + body2.each do |k,v| + k = "@id" if k.eql?('id') + k = "@type" if k.eql?('type') + + next if k.eql?('context') || k.eql?('links') + + assert_equal v, body1[k] + end + end + + + def test_cache_invalidation + # Make a request and cache the response + response1 = @app.get + assert_equal 200, response1.status + + response1 = @app.get + assert_equal 304, response1.status + assert cached?(response1) + + # Invalidate the cache + response2 = @app.get do |req| + req.headers['invalidate_cache'] = true + end + assert_equal 200, response2.status + assert uncached?(response2) + end + + def test_cache_expiration + WebMock.enable! + WebMock.stub_request(:get, @url) + .to_return(headers: { 'Cache-Control': "max-age=1" , 'Last-Modified': Time.now.httpdate}, body: {result: 'hello'}.to_json) + + + # Make a request and cache the response with a short expiry time + response1 = @app.get + assert_equal 200, response1.status + + # Wait for the cache to expire + sleep 2 + + response2 = @app.get + + assert_equal 200, response2.status + assert uncached?(response2) + + sleep 2 + + WebMock.stub_request(:get, @url) + .to_return(headers: { 'Cache-Control': "max-age=100" , 'Last-Modified': Time.now.httpdate}, body: {result: 'hello'}.to_json) + @app.get + + + # Wait for the cache to expire + sleep 2 + + response2 = @app.get + + assert cached?(response2) + + WebMock.disable! + end + + def test_cache_last_modified + WebMock.enable! + # Make a request with Last-Modified header + WebMock.stub_request(:get, @url) + .to_return(headers: { 'Cache-Control': "max-age=1", 'Last-Modified': 3.days.ago.to_time.httpdate}, body: {result: 'hello'}.to_json, status: 304) + + @app.get + + response2 = @app.get + assert cached?(response2) + + sleep 1 + + WebMock.stub_request(:get, @url) + .to_return(headers: { 'Cache-Control': "max-age=10", 'Last-Modified': 1.days.ago.to_time.httpdate}, body: {result: 'hello'}.to_json, status: 304) + + response2 = @app.get + assert refreshed?(response2) + WebMock.disable! + end + + + private + def cached?(response) + response.env.response_headers['x-rack-cache'].eql?('hit') + end + + def uncached?(response) + response.env.response_headers['x-rack-cache'].eql?('miss') + end + + def refreshed?(response) + response.env.response_headers['x-rack-cache'].eql?('fresh') + end +end From 574a045ac1aeede73dcb4e4cbb6b555db04501b3 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Sun, 2 Jun 2024 14:33:37 +0200 Subject: [PATCH 9/9] clean no more ecoportal old code --- lib/ontologies_api_client.rb | 1 - lib/ontologies_api_client/base.rb | 17 ++- lib/ontologies_api_client/models/creator.rb | 12 --- .../models/creator_identifier.rb | 12 --- lib/ontologies_api_client/read_write.rb | 85 ++++----------- lib/ontologies_api_client/utils.rb | 101 ------------------ 6 files changed, 25 insertions(+), 203 deletions(-) delete mode 100644 lib/ontologies_api_client/models/creator.rb delete mode 100644 lib/ontologies_api_client/models/creator_identifier.rb delete mode 100644 lib/ontologies_api_client/utils.rb diff --git a/lib/ontologies_api_client.rb b/lib/ontologies_api_client.rb index 8cce87b..739639a 100644 --- a/lib/ontologies_api_client.rb +++ b/lib/ontologies_api_client.rb @@ -2,7 +2,6 @@ require 'multi_json' require 'spawnling' -require_relative 'ontologies_api_client/utils' require_relative 'ontologies_api_client/config' require_relative 'ontologies_api_client/http' require_relative 'ontologies_api_client/link_explorer' diff --git a/lib/ontologies_api_client/base.rb b/lib/ontologies_api_client/base.rb index 8e2cf8d..aedd782 100644 --- a/lib/ontologies_api_client/base.rb +++ b/lib/ontologies_api_client/base.rb @@ -50,17 +50,12 @@ def self.map_classes end def initialize(options = {}) - begin - values = options[:values] - if values.is_a?(Hash) && !values.empty? - create_attributes(values.keys) - populate_attributes(values) - end - create_attributes(self.class.attrs_always_present || []) - rescue => e - LOGGER.debug("\n\n ECCEZIONE! RUBY API - base.rb - initialize(options): #{e.message}\n#{e.backtrace.join("\n")}") - raise e + values = options[:values] + if values.is_a?(Hash) && !values.empty? + create_attributes(values.keys) + populate_attributes(values) end + create_attributes(self.class.attrs_always_present || []) end def id @@ -157,4 +152,4 @@ def populate_attributes(hash) end end -end +end \ No newline at end of file diff --git a/lib/ontologies_api_client/models/creator.rb b/lib/ontologies_api_client/models/creator.rb deleted file mode 100644 index 4a9569a..0000000 --- a/lib/ontologies_api_client/models/creator.rb +++ /dev/null @@ -1,12 +0,0 @@ -require_relative "../base" - -module LinkedData - module Client - module Models - class Creator < LinkedData::Client::Base - include LinkedData::Client::ReadWrite - @media_type = "http://data.ecoportal.org/metadata/Creator" - end - end - end -end diff --git a/lib/ontologies_api_client/models/creator_identifier.rb b/lib/ontologies_api_client/models/creator_identifier.rb deleted file mode 100644 index c7da7c8..0000000 --- a/lib/ontologies_api_client/models/creator_identifier.rb +++ /dev/null @@ -1,12 +0,0 @@ -require_relative "../base" - -module LinkedData - module Client - module Models - class CreatorIdentifier < LinkedData::Client::Base - include LinkedData::Client::ReadWrite - @media_type = "http://data.ecoportal.org/metadata/CreatorIdentifier" - end - end - end -end diff --git a/lib/ontologies_api_client/read_write.rb b/lib/ontologies_api_client/read_write.rb index 5cc6d98..366f8ba 100644 --- a/lib/ontologies_api_client/read_write.rb +++ b/lib/ontologies_api_client/read_write.rb @@ -1,5 +1,4 @@ require 'active_support/core_ext/hash' -require_relative 'utils' module LinkedData module Client @@ -63,75 +62,29 @@ def cache_refresh(options = {}) end def equivalent?(current_value, new_value) - # LOGGER.debug("\n\n----------------------\nONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::ReadWrite -> equivalent:\n\n > current_value=#{current_value.inspect}\n\n > new_value=#{new_value.inspect}") - begin - # If we're comparing an existing embedded object - # then use the id for comparison - if current_value.is_a?(LinkedData::Client::Base) - #LOGGER.debug(" > current_value is a LinkedData::Client::Base \n > current_value= #{current_value}\n > new_value=#{new_value}") - result_equivalent = current_value.id.eql?(new_value) - #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") - return result_equivalent - end - - # Otherwise, do some complex comparing - case new_value - when String - #LOGGER.debug(" > new_value is a String\n > current_value= #{current_value}\n > new_value=#{new_value}") - result_equivalent = current_value.to_s.eql?(new_value) - #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") - return result_equivalent - when Array, Hash - new_value = nil if new_value.is_a?(Array) && new_value.empty? && current_value.nil? - if new_value.is_a?(Hash) || (new_value.is_a?(Array) && (new_value.first.is_a?(Hash) || new_value.first.is_a?(OpenStruct))) - #LOGGER.debug(" > new_value is an Array && new_value.first is an Hash or OpenStruct") - - ordered_current = LinkedData::Client::Utils.order_inner_array_elements(current_value, true, [:id, :type, :links, :context, :created]) - # LOGGER.debug(" > > ordered_current=#{ordered_current}") + # If we're comparing an existing embedded object + # then use the id for comparison + if current_value.is_a?(LinkedData::Client::Base) + return current_value.id.eql?(new_value) + end - clean_current = LinkedData::Client::Utils.recursive_symbolize_keys(ordered_current, true, [:id, :type, :links, :context, :created]) - # LOGGER.debug(" > > clean_current=#{clean_current}") - - # LOGGER.debug(" > > BEFORE elaboration PROVE: new_value=#{new_value}") - # LOGGER.debug(" > > BEFORE elaboration PROVE: new_value.inspect= #{new_value.inspect}") - # LOGGER.debug(" > > BEFORE elaboration PROVE: new_value_symbolized: #{LinkedData::Client::Utils.recursive_symbolize_keys(new_value)}") - #new_value.map{|e| LOGGER.debug("new_value.map: \n e = #{e}\n e.to_h = #{e.to_h}")} - - #new_value[0].each{ |k,v| LOGGER.debug("new_value[0].each [k,v] = [#{k},#{v}] => #{new_value[0]}")} - - clean_new_to_order = LinkedData::Client::Utils.recursive_symbolize_keys(new_value, true, [:id, :type, :links, :context, :created]) - # LOGGER.debug(" > > clean_new_to_order=#{clean_new_to_order}") - - clean_new = LinkedData::Client::Utils.order_inner_array_elements(clean_new_to_order, true, [:id, :type, :links, :context, :created]) - #LOGGER.debug(" > > clean_new=#{clean_new}") - - result_equivalent = clean_current.eql?(clean_new) rescue clean_current == clean_new - # LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") - return result_equivalent - else - #LOGGER.debug(" > new_value is an Array OR Hash:\n > current_value= #{current_value}\n > new_value=#{new_value}") - result_equivalent = current_value.sort.eql?(new_value.sort) rescue current_value == new_value - #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") - return result_equivalent - end + # Otherwise, do some complex comparing + case new_value + when String + return current_value.to_s.eql?(new_value) + when Array, Hash + new_value = nil if new_value.is_a?(Array) && new_value.empty? && current_value.nil? + if new_value.is_a?(Array) && new_value.first.is_a?(Hash) + clean_current = current_value.map {|e| e.to_h.symbolize_keys.delete_if {|k,v| v.nil?}}.sort_by {|e| e.is_a?(Hash) ? e.values.sort : e} + clean_new = new_value.map {|e| e.symbolize_keys.delete_if {|k,v| v.nil?}}.sort_by {|e| e.is_a?(Hash) ? e.values.sort : e} + return clean_current.eql?(clean_new) rescue clean_current == clean_new + else + return current_value.sort.eql?(new_value.sort) rescue current_value == new_value end - #LOGGER.debug(" > OTHERWISE CASE > current_value= #{current_value}\n > new_value=#{new_value}") - result_equivalent = current_value.sort.eql?(new_value.sort) rescue current_value == new_value - #LOGGER.debug(" > > equivalent RESULT =#{result_equivalent}") - return result_equivalent - rescue => e - LOGGER.debug("\n\n ECCEZIONE! ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::equivalent: #{e.message}\n#{e.backtrace.join("\n")}") - raise e end + return current_value.eql?(new_value) rescue current_value == new_value end - # NEW method Ecoportal - # clean an annidate hash (hash with array, hash or array of hash) - def clean_annidate_hash(obj) - - end - - def invalidate_cache(cache_refresh_all = true) self.class.all(invalidate_cache: true) if cache_refresh_all HTTP.get(self.id, invalidate_cache: true) if self.id @@ -151,4 +104,4 @@ def refresh_cache end end -end +end \ No newline at end of file diff --git a/lib/ontologies_api_client/utils.rb b/lib/ontologies_api_client/utils.rb deleted file mode 100644 index 9bc2b53..0000000 --- a/lib/ontologies_api_client/utils.rb +++ /dev/null @@ -1,101 +0,0 @@ -# --- NEW ECOPORTAL class----- -# Utility class -module LinkedData - module Client - class Utils - - #Symbolize the keys recursively for the object passed as parameter - # Es. {"a" => "aaa", "b" => {"b1" => "bbb1", "b2" => "bbb2"}} - # becomes {:a => "aaa", :b => {:b1 => "bbb1", :b2 => "bbb2"}} - def self.recursive_symbolize_keys(obj, exclude_nil_value = false, excluded_keys = []) - # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::Utils -> recursive_symbolize_keys CALL:") - if obj.is_a?Array - ## LOGGER.debug(" OBJ is an Array") - obj.map do |e| - val = (e.to_h == {}) ? e : e.to_h - recursive_symbolize_keys(val, exclude_nil_value, excluded_keys) - end - else - {}.tap do |h| - obj.each do |key, value| - # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::Utils -> recursive_symbolize_keys [key,value] = [#{key}, #{value}] - key.to_sym= #{key.to_sym}") - # next if excluded_keys.include?(key) - # next if (exclude_nil_value && value.nil?) - h[key.to_sym] = map_value(value, exclude_nil_value, excluded_keys) - end - end - end - end - - def self.map_value(thing, exclude_nil_value = false, excluded_keys = []) - # LOGGER.debug("ONTOLOGIES_API_RUBY_CLIENT: LinkedData::Client::Utils -> map_value: \n > thing = #{thing}") - case thing - when Hash - # LOGGER.debug(" > thing is HASH") - recursive_symbolize_keys(thing, exclude_nil_value, excluded_keys) - when Array - # LOGGER.debug(" > thing is Array") - thing.map { |v| map_value(v, exclude_nil_value, excluded_keys) } - when OpenStruct - # LOGGER.debug(" > thing is OpenStruct") - recursive_symbolize_keys(thing.to_h, exclude_nil_value, excluded_keys) - # when Base - # recursive_symbolize_keys(thing.to_hash, exclude_nil_value) - else - # LOGGER.debug(" > thing type is not recognized") - thing - end - end - - # UTILITY: be an object comparables with another one, indipendently the order of its fields or the order of - # its array elements - # For each array in the object (Hash or OpenStruct) or nested into subobject, this method order the elements. - # That permits to compare two objects (Hash or OpenStruct). - # Es. - # {:a => [3,1,2], :b => {:b1 => [30,10,50], :b2 => [300,100,500]}} - # becomes {:a => [1,2,3], :b => {:b1 => [10,30,50], :b2 => [100,300,500]}} - # - # In case the elements of array are hashs, then it orders them by their hash value - def self.order_inner_array_elements(obj, exclude_nil_value = false, excluded_keys = []) - if obj.is_a?Array - obj.map do |e| - val = (e.to_h == {}) ? e : e.to_h - order_inner_array_elements(val, exclude_nil_value, excluded_keys) - end - #obj.map{|e| order_inner_array_elements(e.to_h, exclude_nil_value, excluded_keys)} - else - {}.tap do |h| - obj.each do |key, value| - next if excluded_keys.include?(key) - next if (exclude_nil_value && value.nil?) - v = map_ordered_value(value, exclude_nil_value, excluded_keys) - h[key.to_sym] = v - end - end - end - end - - def self.map_ordered_value(thing, exclude_nil_value = false, excluded_keys = []) - case thing - when Hash - result = order_inner_array_elements(thing, exclude_nil_value, excluded_keys) - when Array - result = thing.map do |v| - case v - when Hash, Array, OpenStruct - map_ordered_value(v, exclude_nil_value, excluded_keys) - else - v - end - end - result = result.sort_by{|e| e.is_a?(Hash) ? e.hash : e} - when OpenStruct - result = order_inner_array_elements(thing.to_h, exclude_nil_value, excluded_keys) - else - result = thing - end - result - end - end - end -end \ No newline at end of file