From 08961816e76745dd6593f1bcf7ec5a7ebda0e8d9 Mon Sep 17 00:00:00 2001 From: Kenny Synvrit Date: Fri, 4 Feb 2022 17:19:35 -0500 Subject: [PATCH] Add logic to applied sockets to ignore zeros from marketplace Add new field for Marketplace in NormalizedFacts Add new test parameters for marketplace field Add populating logic for isMarketplace. Add logic to applied sockets to ignore zeros for marketplace Create new test case for expected behavior of marketplace instances for RHEL or other products Add logic for ismarketplace in mock host script --- bin/insert-mock-hosts | 100 ++++++++++-------- .../DefaultProductUsageCollector.java | 6 +- .../collector/RHELProductUsageCollector.java | 7 +- .../tally/facts/FactNormalizer.java | 1 + .../tally/facts/NormalizedFacts.java | 1 + .../DefaultProductUsageCollectorTest.java | 26 +++++ .../RHELProductUsageCollectorTest.java | 26 +++++ .../tally/facts/FactNormalizerTest.java | 1 + 8 files changed, 123 insertions(+), 45 deletions(-) diff --git a/bin/insert-mock-hosts b/bin/insert-mock-hosts index 4402650862..9d7e996728 100755 --- a/bin/insert-mock-hosts +++ b/bin/insert-mock-hosts @@ -13,12 +13,11 @@ def generate_host(inventory_id=None, insights_id=None, account_number=None, org_ subscription_manager_id=None, cores=None, sockets=None, is_guest=None, hypervisor_uuid=None, hardware_type=None, measurement_type=None, num_of_guests=None, last_seen=None, product=None, sla=None, usage=None, is_unmapped_guest=None, is_hypervisor=None, cloud_provider=None, - skip_buckets=False, is_hbi=False): - + skip_buckets=False, is_hbi=False, is_marketplace=False): account = account_number or "account123" host_id = uuid.uuid4() - inventory_id=inventory_id or uuid.uuid4() - insights_id=insights_id or uuid.uuid4() + inventory_id = inventory_id or uuid.uuid4() + insights_id = insights_id or uuid.uuid4() if not is_guest: hypervisor_uuid = None @@ -47,6 +46,7 @@ def generate_host(inventory_id=None, insights_id=None, account_number=None, org_ 'cores_per_socket': cores // sockets, 'number_of_sockets': sockets, 'cloud_provider': cloud_provider, + 'is_marketplace': is_marketplace, }), 'stale_timestamp': '2030-01-01', 'reporter': 'rhsm-conduit', @@ -98,29 +98,32 @@ def _generate_insert(table, **host_fields): values = ','.join(db_repr(value) for value in values) output.write(f'insert into {table}({fields}) values ({values});\n') -def _generate_buckets(host_id, product, sla, usage, as_hypervisor=False, measurement_type=None, cores=None, sockets=None): - sla = sla or 'Premium' - usage = usage or 'Production' - bucket_fields = { - 'host_id': host_id, - 'product_id': product or 'RHEL', - 'as_hypervisor': as_hypervisor, - 'measurement_type': measurement_type, - 'cores': cores, - 'sockets': sockets, - } - sla_vals = ["_ANY", sla] - usage_vals = ["_ANY", usage] - for next_sla in sla_vals: - for next_usage in usage_vals: - _generate_insert("host_tally_buckets", sla=next_sla, usage=next_usage, **bucket_fields) +def _generate_buckets(host_id, product, sla, usage, as_hypervisor=False, measurement_type=None, cores=None, + sockets=None): + sla = sla or 'Premium' + usage = usage or 'Production' + bucket_fields = { + 'host_id': host_id, + 'product_id': product or 'RHEL', + 'as_hypervisor': as_hypervisor, + 'measurement_type': measurement_type, + 'cores': cores, + 'sockets': sockets, + } + + sla_vals = ["_ANY", sla] + usage_vals = ["_ANY", usage] + for next_sla in sla_vals: + for next_usage in usage_vals: + _generate_insert("host_tally_buckets", sla=next_sla, usage=next_usage, **bucket_fields) def _create_account_service(account, service_type): - fields = ','.join(['account_number', 'service_type']) - values = ','.join(db_repr(value) for value in [account, service_type]) - output.write(f'insert into account_services ({fields}) values ({values}) ON CONFLICT ON CONSTRAINT account_services_pkey DO NOTHING;\n') + fields = ','.join(['account_number', 'service_type']) + values = ','.join(db_repr(value) for value in [account, service_type]) + output.write( + f'insert into account_services ({fields}) values ({values}) ON CONFLICT ON CONSTRAINT account_services_pkey DO NOTHING;\n') parser = argparse.ArgumentParser(description='Insert mock hosts into the a local DB') @@ -178,6 +181,8 @@ parser.add_argument('--opt-in', action='store_true', help='Opt in this account') parser.add_argument('--clear', action='store_true', help='Clear data in the hosts table and related tables') +parser.add_argument('--marketplace', action='store_true', dest='is_marketplace', + help='Set a system as a Marketplace instance') args = parser.parse_args() if args.is_hbi: @@ -191,7 +196,7 @@ os.environ['PGPASSWORD'] = args.db_password or default_password psql = None if not args.output_sql: - psql = subprocess.Popen('psql', stdin=subprocess.PIPE, shell=True, text=True) + psql = subprocess.Popen('psql', stdin=subprocess.PIPE, shell=True, encoding='utf-8') output = psql.stdin output.write('begin;') @@ -207,39 +212,49 @@ for i in range(args.num_accounts): account = args.account if args.opt_in: _generate_insert('account_config', - account_number=account, - sync_enabled=True, - reporting_enabled=True, - opt_in_type='API', - created='2021-01-01', - updated='2021-01-01' - ) + account_number=account, + sync_enabled=True, + reporting_enabled=True, + opt_in_type='API', + created='2021-01-01', + updated='2021-01-01' + ) product = args.product or "RHEL" for i in range(args.num_aws): generate_host(account_number=account, hardware_type='VIRTUAL', measurement_type='CLOUD', - product=args.product, sla=args.sla, usage=args.usage, cores=args.cores, sockets=args.sockets, is_hbi=args.is_hbi, - cloud_provider='AWS') + product=args.product, sla=args.sla, usage=args.usage, cores=args.cores, sockets=args.sockets, + is_hbi=args.is_hbi, + cloud_provider='AWS', is_marketplace=args.is_marketplace) for i in range(args.num_physical): generate_host(account_number=account, hardware_type='PHYSICAL', measurement_type='PHYSICAL', - product=args.product, sla=args.sla, usage=args.usage, cores=args.cores, sockets=args.sockets, is_hbi=args.is_hbi) + product=args.product, sla=args.sla, usage=args.usage, cores=args.cores, sockets=args.sockets, + is_hbi=args.is_hbi, + is_marketplace=args.is_marketplace) hypervisor = None for i in range(args.num_hypervisors): - hypervisor = generate_host(account_number=account, hardware_type='PHYSICAL', measurement_type='VIRTUAL', product=args.product, - sla=args.sla, usage=args.usage, cores=args.cores, sockets=args.sockets, is_hypervisor=True, is_hbi=args.is_hbi) + hypervisor = generate_host(account_number=account, hardware_type='PHYSICAL', measurement_type='VIRTUAL', + product=args.product, + sla=args.sla, usage=args.usage, cores=args.cores, sockets=args.sockets, + is_hypervisor=True, is_hbi=args.is_hbi, + is_marketplace=args.is_marketplace) # create a physical entry for the host as well. - generate_host(account_number=account, hardware_type='PHYSICAL', measurement_type='PHYSICAL', product=args.product, - sla=args.sla, usage=args.usage, cores=args.cores, sockets=args.sockets, is_hypervisor=True, is_hbi=args.is_hbi) + generate_host(account_number=account, hardware_type='PHYSICAL', measurement_type='PHYSICAL', + product=args.product, + sla=args.sla, usage=args.usage, cores=args.cores, sockets=args.sockets, is_hypervisor=True, + is_hbi=args.is_hbi, + is_marketplace=args.is_marketplace) create_unmapped = args.unmapped_guests for i in range(args.num_guests): if args.hypervisor_id: hypervisor_uuid = args.hypervisor_id elif hypervisor is not None: - hypervisor_uuid = hypervisor.get('subscription_manager_id') or json.loads(hypervisor['canonical_facts'])['subscription_manager_id'] + hypervisor_uuid = hypervisor.get('subscription_manager_id') or json.loads(hypervisor['canonical_facts'])[ + 'subscription_manager_id'] # If we are associating the guests with a created hypervisor host, don't allow it to be unmapped # since it was considered as reported. create_unmapped = False @@ -248,9 +263,10 @@ for i in range(args.num_accounts): skip_buckets = "RHEL" in product.upper() generate_host(account_number=account, hardware_type='VIRTUALIZED', is_guest=True, - product=args.product, sla=args.sla, usage=args.usage, hypervisor_uuid=hypervisor_uuid, - cores=args.cores, sockets=args.sockets, is_unmapped_guest=create_unmapped, - skip_buckets=skip_buckets, is_hbi=args.is_hbi) + product=args.product, sla=args.sla, usage=args.usage, hypervisor_uuid=hypervisor_uuid, + cores=args.cores, sockets=args.sockets, is_unmapped_guest=create_unmapped, + skip_buckets=skip_buckets, is_hbi=args.is_hbi, + is_marketplace=args.is_marketplace) output.write('commit;') diff --git a/src/main/java/org/candlepin/subscriptions/tally/collector/DefaultProductUsageCollector.java b/src/main/java/org/candlepin/subscriptions/tally/collector/DefaultProductUsageCollector.java index 381ad7625a..0fa7dccc74 100644 --- a/src/main/java/org/candlepin/subscriptions/tally/collector/DefaultProductUsageCollector.java +++ b/src/main/java/org/candlepin/subscriptions/tally/collector/DefaultProductUsageCollector.java @@ -38,18 +38,22 @@ public Optional collect( HardwareMeasurementType appliedType = null; // Cloud provider hosts only account for a single socket. if (normalizedFacts.getCloudProviderType() != null) { - appliedSockets = 1; + appliedSockets = normalizedFacts.isMarketplace() ? 0 : 1; appliedType = normalizedFacts.getCloudProviderType(); prodCalc.addCloudProvider( normalizedFacts.getCloudProviderType(), appliedCores, appliedSockets, 1); } // Accumulate for physical systems. else if (!normalizedFacts.isVirtual()) { + appliedSockets = normalizedFacts.isMarketplace() ? 0 : appliedSockets; appliedType = HardwareMeasurementType.PHYSICAL; prodCalc.addPhysical(appliedCores, appliedSockets, 1); } // Any other system is considered virtual else { + if (normalizedFacts.isMarketplace()) { + appliedSockets = 0; + } appliedType = HardwareMeasurementType.VIRTUAL; prodCalc.addToTotal(appliedCores, appliedSockets, 1); } diff --git a/src/main/java/org/candlepin/subscriptions/tally/collector/RHELProductUsageCollector.java b/src/main/java/org/candlepin/subscriptions/tally/collector/RHELProductUsageCollector.java index 5158845c3c..f731d48f3a 100644 --- a/src/main/java/org/candlepin/subscriptions/tally/collector/RHELProductUsageCollector.java +++ b/src/main/java/org/candlepin/subscriptions/tally/collector/RHELProductUsageCollector.java @@ -47,7 +47,7 @@ public Optional collect( // Cloud provider hosts only account for a single socket. if (normalizedFacts.getCloudProviderType() != null) { - appliedSockets = 1; + appliedSockets = normalizedFacts.isMarketplace() ? 0 : 1; prodCalc.addCloudProvider( normalizedFacts.getCloudProviderType(), appliedCores, appliedSockets, 1); return Optional.of( @@ -61,7 +61,7 @@ public Optional collect( // If the hypervisor is unknown for a guest, we consider it as having a // unique hypervisor instance contributing to the hypervisor counts. // Since the guest is unmapped, we only contribute a single socket. - appliedSockets = 1; + appliedSockets = normalizedFacts.isMarketplace() ? 0 : 1; prodCalc.addHypervisor(appliedCores, appliedSockets, 1); return Optional.of( createBucket( @@ -70,6 +70,9 @@ public Optional collect( // Accumulate for physical systems. else if (!normalizedFacts.isVirtual()) { // Physical system so increment the physical system counts. + if (normalizedFacts.isMarketplace()) { + appliedSockets = 0; + } prodCalc.addPhysical(appliedCores, appliedSockets, 1); return Optional.of( createBucket( diff --git a/src/main/java/org/candlepin/subscriptions/tally/facts/FactNormalizer.java b/src/main/java/org/candlepin/subscriptions/tally/facts/FactNormalizer.java index 55047f53e7..a51fbc9187 100644 --- a/src/main/java/org/candlepin/subscriptions/tally/facts/FactNormalizer.java +++ b/src/main/java/org/candlepin/subscriptions/tally/facts/FactNormalizer.java @@ -223,6 +223,7 @@ private void normalizeMarketplace(NormalizedFacts normalizedFacts, InventoryHost return; } + normalizedFacts.setMarketplace(hostFacts.isMarketplace()); if (normalizedFacts.getCores() != 0) { normalizedFacts.setCores(0); } diff --git a/src/main/java/org/candlepin/subscriptions/tally/facts/NormalizedFacts.java b/src/main/java/org/candlepin/subscriptions/tally/facts/NormalizedFacts.java index 39eb5ee3a7..041cf2a946 100644 --- a/src/main/java/org/candlepin/subscriptions/tally/facts/NormalizedFacts.java +++ b/src/main/java/org/candlepin/subscriptions/tally/facts/NormalizedFacts.java @@ -53,6 +53,7 @@ public class NormalizedFacts { private boolean isVirtual; private boolean isHypervisor; private boolean isHypervisorUnknown; + private boolean isMarketplace; private HostHardwareType hardwareType; private HardwareMeasurementType cloudProviderType; diff --git a/src/test/java/org/candlepin/subscriptions/tally/collector/DefaultProductUsageCollectorTest.java b/src/test/java/org/candlepin/subscriptions/tally/collector/DefaultProductUsageCollectorTest.java index 8864d8a5b4..67dca77f48 100644 --- a/src/test/java/org/candlepin/subscriptions/tally/collector/DefaultProductUsageCollectorTest.java +++ b/src/test/java/org/candlepin/subscriptions/tally/collector/DefaultProductUsageCollectorTest.java @@ -23,6 +23,8 @@ import static org.candlepin.subscriptions.tally.collector.Assertions.*; import static org.candlepin.subscriptions.tally.collector.TestHelper.*; +import java.util.LinkedList; +import java.util.List; import org.candlepin.subscriptions.db.model.HardwareMeasurementType; import org.candlepin.subscriptions.db.model.ServiceLevel; import org.candlepin.subscriptions.db.model.Usage; @@ -104,6 +106,30 @@ public void testCountsForCloudProvider() { assertNullExcept(calc, HardwareMeasurementType.TOTAL, HardwareMeasurementType.AWS); } + @Test + void testCountsForMarketplaceInstances() { + // Marketplace instance zeros should be ignored from the overall total + List conditions = new LinkedList<>(); + NormalizedFacts marketFacts = cloudMachineFacts(HardwareMeasurementType.AWS, 1, 0); + marketFacts.setMarketplace(true); + conditions.add(marketFacts); + + NormalizedFacts physicalNonHypervisor = physicalNonHypervisor(1, 0); + physicalNonHypervisor.setMarketplace(true); + conditions.add(physicalNonHypervisor); + + NormalizedFacts virtual = guestFacts(5, 0, false); + virtual.setMarketplace(true); + conditions.add(virtual); + + UsageCalculation calc = new UsageCalculation(createUsageKey()); + + for (NormalizedFacts current : conditions) { + collector.collect(calc, current); + } + assertTotalsCalculation(calc, 0, 0, 3); + } + private UsageCalculation.Key createUsageKey() { return new UsageCalculation.Key("NON_RHEL", ServiceLevel.EMPTY, Usage.EMPTY); } diff --git a/src/test/java/org/candlepin/subscriptions/tally/collector/RHELProductUsageCollectorTest.java b/src/test/java/org/candlepin/subscriptions/tally/collector/RHELProductUsageCollectorTest.java index b3b78928a5..f6bc91b7af 100644 --- a/src/test/java/org/candlepin/subscriptions/tally/collector/RHELProductUsageCollectorTest.java +++ b/src/test/java/org/candlepin/subscriptions/tally/collector/RHELProductUsageCollectorTest.java @@ -24,6 +24,8 @@ import static org.candlepin.subscriptions.tally.collector.TestHelper.*; import static org.junit.jupiter.api.Assertions.*; +import java.util.LinkedList; +import java.util.List; import java.util.Optional; import org.candlepin.subscriptions.db.model.HardwareMeasurementType; import org.candlepin.subscriptions.db.model.HostTallyBucket; @@ -122,6 +124,30 @@ void testCountsForCloudProvider() { assertNullExcept(calc, HardwareMeasurementType.TOTAL, HardwareMeasurementType.AWS); } + @Test + void testCountsForMarketplaceInstances() { + // Marketplace instance zeros should be ignored from the overall total + List conditions = new LinkedList<>(); + NormalizedFacts marketFacts = cloudMachineFacts(HardwareMeasurementType.AWS, 1, 0); + marketFacts.setMarketplace(true); + conditions.add(marketFacts); + + NormalizedFacts physicalNonHypervisor = physicalNonHypervisor(1, 0); + physicalNonHypervisor.setMarketplace(true); + conditions.add(physicalNonHypervisor); + + NormalizedFacts virtual = guestFacts(1, 0, true); + virtual.setMarketplace(true); + conditions.add(virtual); + + UsageCalculation calc = new UsageCalculation(createUsageKey()); + + for (NormalizedFacts current : conditions) { + collector.collect(calc, current); + } + assertTotalsCalculation(calc, 0, 0, 3); + } + private UsageCalculation.Key createUsageKey() { return new UsageCalculation.Key("RHEL", ServiceLevel.EMPTY, Usage.EMPTY); } diff --git a/src/test/java/org/candlepin/subscriptions/tally/facts/FactNormalizerTest.java b/src/test/java/org/candlepin/subscriptions/tally/facts/FactNormalizerTest.java index 7be4462f4a..8b74ab4083 100644 --- a/src/test/java/org/candlepin/subscriptions/tally/facts/FactNormalizerTest.java +++ b/src/test/java/org/candlepin/subscriptions/tally/facts/FactNormalizerTest.java @@ -669,6 +669,7 @@ void testCalculationOfMarketplaceMeasurements() { facts.setSystemProfileCoresPerSocket(7); facts.setSystemProfileSockets(1); NormalizedFacts normalizedFacts = normalizer.normalize(facts, Collections.emptyMap()); + assertTrue(normalizedFacts.isMarketplace()); assertEquals(0, normalizedFacts.getCores()); assertEquals(0, normalizedFacts.getSockets()); }